diff --git a/.gitattributes b/.gitattributes index 3d742cd27f2e95718ffb6b2f9b0f531d54c24e28..6e6f5012e409a3b7a38e8c21bb68ab25d74f799e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -249,3 +249,51 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 221m60b4b8/evaluation/generation/examples.221m60b4b8_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text 280m5b91b5/evaluation/generation/examples.280m5b91b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text 1b191b91b/evaluation/generation/examples.1b191b91b_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c8f4e9639c1ee3695811945afddf2389d665924 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22d812fad8367a673e6e1045da2c7c735650033d0d88efd3a4fa0912e166e6e9 +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05bb06123eb76897db3b79e3568efed56aab3064 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7ae33484dfdc0edf5260d45d0d1fdd16f88df9695a70d663436f317e47a161 +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69f34f8b2c799fc85ba6152f92952de6e4992f15 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e08491890babe9e7eac57d2be46f3d40d853e545fbe6551a486b541b43486ec +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c3afb8e83ed0f433aefce05fecb3eff97b382b9 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c424295d30c6b1134d8910b346f41911eeb20f8d0041a229d1eaabbb9431b477 +size 116013602 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df0da9f28eacad29b11e573f84d5d075d9150b5a --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b7b5a2b386177b2cca060078edbab76375869a69db802d6b1b6fddce593123 +size 116013858 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39d0970e4c2fa8d1bc5b9e612d0b3b150b2b86e5 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791617a832366eeae28f2b5a0d85fe4655bc1c9782c8a13b9fe4fa91d649e3d0 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa53d4ae93a4d32e948a674ee104fe146894ad31 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c884e49421ecef839753a9a7db5aaabc8cf5fc6868a6196779b9bfe3b9a9662a +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb639591b6e1826a471f2d992bdf8f750193c57b --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120bfd18f1a97cb8857ed2d0d64b4f8d26d4e20116df96383ffc60a9b94af247 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bcfd3083031a773957c13adbc98852ba1e69c30 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaf8814f5a975bfb43dbca86edb40ff3dcd0feb5dc0b899532a41efb0bed4df1 +size 116013602 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6132ec8ea0f1f826d41dcd93f7923cc9a8bf167 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b1ac06c41b2209befbd7ccafd48b2a35b445ecce385b973d826b7c447ab91e4 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0477388c3e2e86b413a16b49ed3714c53cfa8139 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1802c85ec3938cb8f077c99855781e9fe67f9a6a9e6ebd5672c0407cdd67d3ac +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90eb61b22a10bca2f9769bd2303f78b5531253bc --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de78540ebcfc3665f9feb1df17dbaa333d1ad12790fa6ec38819a0487c0af716 +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6812043eacfa14ddd4ceb83d81f3f77f4385d0af --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00bb2fd5c39f89ef2994aba27d90b0f58caf5a2ae75e348ad9a6e56641b811d0 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf8e7cb022e5c65ad5ad31abe75016a14dc57e24 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4db8ab4c8d2a1122bed580ec0298490103cae967c956278524d1f49e01da0d +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06fd74d54fc969a70807908b9a855eda741c70e3 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c652b052e1e58d4e847e634c84395d7862ce49c21a8f2c8ed813ca54fd233eb +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6de2d6ed668fd0c82727ee3fc78e67818bbace6 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47a336a40aa09d896ec46f99d613c07df5bc2ce63b53088a1ade6704e611e929 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c85b52fcc9a0f5969e5a3a2861e2dc9ff2a879 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:167027af5f668c4f583671f4cb5d8fe4b0284ea72046523e0bf349e6d4d9c540 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb13eef95544ca573c4f8dd989811b1c9eecdbc5 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ddbfc35072259979549cd7b5131b32b699a889bd259067f3f579c1f463e2591 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bae3290e5df7f86c08aae0a8c9f82753e5fa9346 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4273ed7c0b16f3708dd51f792df3e35c9e87cd00e7c79c9f16ece4692fa0aa6 +size 116013858 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e777570530e18e64bf985093c61a26730872e042 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:089b06f7d188ca171e2ccc5198e93bf0a22ebc5ca50719f0e98171a13bcb4815 +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a1fd334002516de56760d14ebb05f176b2ed4e9 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f70e9783b95177ac1e9715688c92242b4e611e49d8b6b689fa5e7e8e2cd578 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8932fc9443fd650b1c60390a162c587b5b2d21d8 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a57caf4cf52c6a8ad06c799b70313b6ae8d64bbffa74528b9bd0bdc43746611 +size 116013602 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31e1c360b8298eb1e56516927f762852bf280e18 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9083b70ae70014bfd5935e9edc26ccc22cdc2eef99bda06f72842f80125e9612 +size 116013655 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..765dafff48374aba38c36222f8056b1111f1fa52 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:403ed5edddd0fa7e09f50746ac96cc97854e59fe05cfe670800613d989d9f453 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b635458ab9ff30649b6446675dde9a070d7e1357 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eb29a57f28896fbf9493f38a557de7b14e011519f91e69550ef14bcae479a8f +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8fe3c6db5f31f9f55a30d1ef3cf183201c80a7d --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82927597650e2e7359d7b6d41588e81938e405a8a7483ab84cc81c0bb6449834 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1331f20e9e5428f060fde43d2324a4ed5df6fa4 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13bad050098e899412c69dfdad358a492a9f3cb003711c68dd0ae07dc37e6d27 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06b2b997029c7c3f889b775b694eb5fda9025073 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610cbdbcc97cbeee6c929ca5589427bc6d2b847cf2e03d1c297a98c09028b7d8 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96e544438013293799974b493c0b32ad1006c0e1 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a968eb8af09fdd8028a17316eeca2c6bc2508c898977b0a41a716a7fc63d41f +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fefe8256ca90525c70c5524ea7652f470cd5cf16 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:272b14fd07f61d1e858f150653c0d8cc08d2fc9a950e4363357809adc1434954 +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09b8097818ab6a0bbbf4c9f7856e3608b6778792 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5466cd29761b02a1ab6f9490a9c3a31c3634ab728993cb18cc4fbb370ffd849 +size 116013858 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d772942613e4a17da4c053183cd2c670d5eeb6a0 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b3d897ece6d9f6676a401e78608afc8d586ac1e28560722bdc4f44507c21f7 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..420b5a65624b6bc994c7d440915a8eefab53412d --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc73c67b460d8d6660ed2bc2d4b75e96f44b6882f99bc2c894749e2a23231b7d +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e527d3d69030dfdbc986db78b05d2f8f51c11359 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:876d9512ff25cced375c0212b27da8028af047ac27d7a47a10205aeca97b35d6 +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..725bdfd57f3d21d6e623a43c1b4533feb7d4e358 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11bb67e47494b05de5d0c9d55e01a736aa75535d2960b5943949d9103564d2ec +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be78d1db580ab8ae0792edb5c84750ea4c0bdb14 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe8e75d7ba8e41fab5eb2f8e83bbdbd783945cb3a9c99027d61a1d8b2f6a2e0f +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c7b537a7f51525809707cdc1eb4c00960bfc029 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e93c98cb5644ce116e77673eec03b4b48144d05d5239c636636000ba23b79d28 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a83d5dc2f569f55c00f71e6f0e29604feded40b --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d00355bfb0427ed753d1c3dfa50a56c600b66a2500c7629f41e8ff578d2cb9e3 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1834de94ff5042dc5da95d053e7189430fddab7 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ca95be9ecd7dd23004aa1198f27eb25fe17fdc755d39e1a622692d73d7f05f +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaca3019c29c5da8bfdd5505c09060f8dbee73c2 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27874c189239d90c110546bee0230959f21532a3ed0a80dfc4bda04887d1e25 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..415b41e1b3a8de7b11c5426487fe213b67bd0709 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e762375fc1a6b7b6b801963276decebca970e9f751d7e8976fc3659e477d2d1 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2191c292756a2a6df879fcac9d16c8d1dd422199 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26abdf393a79d4a1b5f0a5fef21caf0f5390c97610d9113980d298a8700c6efa +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d92715da842f80d29a9d3c57fb97355f069cf26 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2d23b3137b21a79d270c1d6806e1378bef8ce20b2f552910f97810a48e3a3ce +size 116013922 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..761c5859f2f0f283b145b4a4ade1c6f102fe53aa --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9293802ecd2b312a119ecd0afacff1d66d313f11ecc26083be0bcf2e0ce2bbdb +size 116013538 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18b1cf49f5a9c2df18cc6c247f5a5a538537a03f --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03dbcf025829a26d400b20072f446fb932210b05347853690105e3ed6e20059a +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de88cb1a88b34f267fc2f94e310afe1fa2438708 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54b1307d4bea5cad4c467db488611b1ffb0baea462a99287d944ccc4a33c1f2a +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ffed93d4616c3246cfdbd740c677cb0ab4f87f5 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e6349a17bef1aa506ed90f7ae6d90978824e311baa0d87cebc951e862b52909 +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a82633811d21c0a60a2a265982e7c487f9be1a0a --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aa23b6be274278c643c2ec6184bb680a0796a8bf22de7f3eb5b8620e9d6089c +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a0045ea6b732381d0e3ddc214ba0b45ee5a3159 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70fd72fad5d4efc1f648a513b5ca6c43f04c905e8ed44a678d3bae9d9bcc3d1c +size 116013922 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c2f117b796a98a275dc698055525a8f2455ba6d --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27becd832ece90c62d14b4273fac529647598f9ed4356d9f96bcf1df93cd07dc +size 116013602 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85826bee8db9aa59fd2184981c3fc7cb4e1844f9 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:223236df7bcb3523bdac80eab9737a5908b933ad551fbff1ea1e6f8b010fb44f +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7513979ffa8f7d24a113cd9080fb82ce51059f1 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1a2e3e74ad79b631f0f613ae5b0b4e835ad05b40f60666f2445087a1820acb2 +size 116013602 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0023db84e9006d3969fe7411cc8b1c3df82e07b7 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d6e09306b5184e7e990dcfaf1d73f36a44283c4edb28c78510e607537e97ad +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dc514c48394b3fde909421334be75dbf85bedc8 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a108d2ee22dd753e1412e95d07a388eb915737860e2db1a398ee4c71c5d0d157 +size 116013858 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a58a422d88a07940a9dacdab9adab71ba8d89e8b --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b820cf6a8f4c41484f7a616dcc8d68e98cada5f229d1afdc56976e7808685f0 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31bc065e6d0987dbf7d9bf43c680c4ce9bc04bba --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d71e53e51bca05e69c230d144e17b82b73884983aeda2fc29edc6fb734a9298 +size 116013655 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35376ff0154a14372545a2f70bdc3e9d03636845 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bee5d91b4fb0ef66e870a84cd3e6e666c3d09f3d615b1db5121cd593a46f763e +size 116013730 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bfb252eb6fcc6283e89c28832901d84a7ff55d4 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61300afde10ba1a462986b7f71fe5220d72f1a19e00c5a2ae6647b20266a3dd6 +size 116013794 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a03b9f8c8102d8e82bb59a32cfb14bb856e879 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b15214f03246828a4188634c8a8a08a32a397113ffab03ef8c59c0babb973d35 +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ebea02bc92e893f3110d6c9b4e4dd7bc874fe9 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f9eb94f2401774fd061c92795d77125b87dcbe338b94f150a27557c3177471f +size 116013666 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6aeee3dd95e04d7fbdfb4f8e9a7af2e0274bdecb --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ded5b9bcf929438c5b1294daa8ef5bddecce2cf8e08230ea777aa337e726159 +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..373052cd180cd0bbc22e921ce46d83bb7a1644fc --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a4e1c2a2bd1c4c5e683f47bddfc5a7b236a6884e690d8fbc74363d282ef52b0 +size 116013719 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d8248bb14805d978a47c97f9fba1205fd28f01e --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e86db0ad7ed5355100dd3b8727e2aff765435a9cac3c8cb7972312c34cbfeeb +size 116013655 diff --git a/619m2b7100m/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m2b7100m/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cee9d2f0070d08a3d97e603c5c66fe81cdfbcf94 --- /dev/null +++ b/619m2b7100m/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e9819492ccaf9f5a9626f627988a8de620373eae4276f4b74e33d71644f4f6 +size 116013591 diff --git a/619m2b7100m/global_step5111/layer_01-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4171c28f83a203a64061894e0a8f1750e0d013f --- /dev/null +++ b/619m2b7100m/global_step5111/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23564823082f636d38cedd57f66ca2d3fb78a5be813276f9aba9855c8a20b1d +size 160826627 diff --git a/619m2b7100m/global_step5111/layer_03-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c0ce91650882bceff8b30435ea3f86187015552 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45757554e0635d3c68dae46be83077c706e1d133f812f10862362c8077464ac4 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_04-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af56488be4b6bf608e578656e30edf09ee073bec --- /dev/null +++ b/619m2b7100m/global_step5111/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8860286776e9a4880eff11137d44bf0f8cb922234ee59196d21182087b0ff5 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_05-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b2a8021c983aa78d78e9e8a187508734abb9771 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62c9450baef221fb919554ad10a79fb4c615c90751be5d55b2017635645e7e02 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_06-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3c4d9d8493fcce99eabd6e2c14530f9966ac97a --- /dev/null +++ b/619m2b7100m/global_step5111/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765f4aee2442d7ba0a332b8bb1da7115d92e53fe6c227d18acc5842a7387f591 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_07-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37812e6201fe824430519bbebef9ad6b0ae87da0 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26910f65717b5927f8857f02a176e2aa6777b8e36562889ded892866f22c8b7e +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_08-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a65222762c8b049ead23f9c714f15b9fe1631d4 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69303df927dd1fd9a5b0d5ea68afd91c84fea05c2067af74b079cd86d1538ed6 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_09-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0732f42b5689f663927765a69abaee7aa62b2da --- /dev/null +++ b/619m2b7100m/global_step5111/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325834f7d1e034a3d399849d23a4da261fba8130f0b490a4890eb645a8ebaa37 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_10-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f44d0ae2969b1f30a9661f731f9e860f0e54bdb5 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85d014b6238592f05da77c564b64927803457e423fc54309cde55536abd9147 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_11-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ec5023236435b7ea34909b0b85277a100592f38 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faecf313620128baa1f9763176affe64305449455b1788de394356cba80fef72 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_12-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..649e13be010aae04139e886ab1fb5d0afbe19463 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:846185d59611a30ebc840ab1efe26edf7c974ae7e4050a65565ce8b4ee6c297e +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_13-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90872cf55a2fcc55652bade622ab38802e3c87e1 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5bbc71dbf88ec059aa68896c4d4ddcc04a562dbe3a03110707fd733f688b37 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_14-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ea2b4572d7e2d02ca5eed8cfc04a8e30eb22897 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:481136593761fa539fb4d887094fe996f7283ec489c92d15263b0a8a64fc9de2 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_15-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5517e11336e15b768ef9ee42ac4a12056a63a3ea --- /dev/null +++ b/619m2b7100m/global_step5111/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeabe01977cfde23dd6c22aadf7a7692d86337c20cdedbc2b4c73f420f387962 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_16-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93ca90d81e22f408a3afe2d4fa498dcd8cb340f4 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc752cf19ca7f936ff3498681c2236cad854823dc7085662413f3e6b37b98f4 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_17-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9d323b99dc524fae6d289ac36a2f42612d43226 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5685bdb88122a746c7da073ad64060a3cf4d5b2b25ebf5dfb75fb7d08e7ede +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_18-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50392a7f8c1ce68cd6aa17bccbd51c73b4ece48a --- /dev/null +++ b/619m2b7100m/global_step5111/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:005f497478e2ca29f5a892887af7f66d2d9ad614baf43275401d5a8c42d0518e +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_19-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92a80c1d347795d79bfd68812093f1886d583208 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7edd682542f7a85e4ca03526c9d34204e36414f23f753017a2aabcb6626760db +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_20-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ed948992359398f1515718aeda3b6eb0b88440a --- /dev/null +++ b/619m2b7100m/global_step5111/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba158fcd7c8fb2d765f4200bf9dc69b9a986be7fc60f1e6b16af79b87dafc2fa +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_21-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57b838cce233f40a1622f995097a809791cefdd1 --- /dev/null +++ b/619m2b7100m/global_step5111/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d337fe92fe9132207a85e489ff54f9ce968865f4ec01df6a635e646cfe14e2 +size 56667395 diff --git a/619m2b7100m/global_step5111/layer_23-model_00-model_states.pt b/619m2b7100m/global_step5111/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af4f314f746915b66203ab4f13ef407f124ac03e --- /dev/null +++ b/619m2b7100m/global_step5111/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbe3b7bf31c03a0f3e41cad99d4ba5d1dc0a57f02b560c24b878c546e6f7be2 +size 7363 diff --git a/619m2b7100m/global_step5111/mp_rank_00_model_states.pt b/619m2b7100m/global_step5111/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56f85be594ca6ff74041775313be7edaef8d05a8 --- /dev/null +++ b/619m2b7100m/global_step5111/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:668a4baf1b18cf17c6b90b1fa3597ea65b05436b640ee5cd6fbe0c0f202b2e53 +size 38515 diff --git a/619m2b7100m/logs/3165618.err b/619m2b7100m/logs/3165618.err new file mode 100644 index 0000000000000000000000000000000000000000..58f57131f48e6c2a72200f654f971eacd756c3a9 --- /dev/null +++ b/619m2b7100m/logs/3165618.err @@ -0,0 +1,1124 @@ +6: 2023-02-28 00:00:18.652613: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652626: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652627: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652621: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652823: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652828: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652843: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: 2023-02-28 00:00:18.652634: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652627: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653020: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653023: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652828: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652841: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 00:00:18.652640: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653043: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653045: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652831: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:18.652824: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: 2023-02-28 00:00:18.653128: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653142: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653130: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: 2023-02-28 00:00:18.653046: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653036: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653145: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653132: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 00:00:18.653056: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653131: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 00:00:18.653127: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724715: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724716: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724727: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: 2023-02-28 00:00:18.724800: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724804: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724811: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724712: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724718: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724799: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724734: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724735: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724820: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724830: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 00:00:18.724745: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 00:00:18.724825: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725143: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725156: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725158: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 00:00:18.725162: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752448: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752457: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752462: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752463: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752455: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752465: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752467: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 00:00:18.752478: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 00:00:20.856248: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856441: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856441: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856445: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:20.856449: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856449: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856451: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856453: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 00:00:20.856462: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.857775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-28 00:00:20.857784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857788: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.857782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-28 00:00:20.858152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.858128: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858130: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858133: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858135: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:20.858136: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858138: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858142: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 00:00:20.858142: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:20.858347: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858353: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858352: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858354: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858355: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858357: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858362: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 00:00:20.858367: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.860885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.860903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:20.861269: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861273: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861275: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861277: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861282: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861283: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861286: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 00:00:20.861289: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861413: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861414: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861435: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:20.861805: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861807: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861812: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861816: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 00:00:20.861815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.861702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.861711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:20.862040: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862042: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862044: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862047: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862047: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 00:00:20.862052: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899398: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899411: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899405: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899405: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899416: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-28 00:00:20.899409: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899517: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899518: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899519: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899524: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:20.899525: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899527: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899532: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 00:00:20.899531: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899407: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:20.899812: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899811: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899819: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899821: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899825: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:20.899826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 00:00:26.231566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.231673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.231669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.231682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.231804: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.231680: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231580: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.231679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.231809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.231684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.231812: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.231685: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.231588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.231816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.231688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: 2023-02-28 00:00:26.231819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.231823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: 2023-02-28 00:00:26.231827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: 2023-02-28 00:00:26.231832: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231951: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.231959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232339: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.232349: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232987: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232990: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.232997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233799: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 00:00:26.233910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233922: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-28 00:00:26.233802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.234013: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-28 00:00:26.233802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233929: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233925: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.233934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.234014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233811: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 00:00:26.233812: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233812: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 00:00:26.233816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 00:00:26.233817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233931: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.234015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-28 00:00:26.233819: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 00:00:26.233820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233822: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.234016: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 00:00:26.233840: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.233939: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.234019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 00:00:26.233918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233928: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.233943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: 2023-02-28 00:00:26.234019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 00:00:26.233934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233935: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233936: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 00:00:26.233938: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 00:00:26.233940: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 00:00:26.234021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.234031: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 00:00:26.234031: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-28 00:00:26.234031: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 00:00:26.234034: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 00:00:26.234034: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.234035: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 00:00:26.234039: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-28 00:00:26.234064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 00:00:26.234077: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234253: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234255: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234267: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234266: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234269: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234270: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 00:00:26.234271: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: 2023-02-28 00:00:26.234439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 00:00:26.234451: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234372: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234376: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234379: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234381: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 00:00:26.234388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 00:00:26.234394: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234970: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.234984: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234985: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234994: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234993: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.234994: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.235001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.235007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 00:00:26.235016: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 00:00:26.235021: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.273878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.273919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235960: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.235955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235966: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.235957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275905: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 00:00:26.235966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235977: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 00:00:26.235980: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.235981: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.235983: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 00:00:26.235984: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 00:00:26.235984: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275909: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275928: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 00:00:26.275989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 00:00:26.275990: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +3: Successfully preprocessed all matching files. +3: Successfully preprocessed all matching files. +3: Successfully preprocessed all matching files. +6: Successfully preprocessed all matching files. +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: +1: +1: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: +2: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +4: +4: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Loading extension module utils...Loading extension module utils...Loading extension module utils... +0: +0: +0: Loading extension module utils... +0: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils...Loading extension module utils... +3: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils... +3: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils...Loading extension module utils... +4: +4: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +5: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m2b7100m/logs/3165618.out b/619m2b7100m/logs/3165618.out new file mode 100644 index 0000000000000000000000000000000000000000..1b844f0c4edc7cdd550c98f4f7f0509b15b048eb --- /dev/null +++ b/619m2b7100m/logs/3165618.out @@ -0,0 +1,6693 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m2b7100mval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_619m2b7100mval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m2b7100m --load checkpoints_619m2b7100m --train-weighted-split-paths-path train100m.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3165618.json --zero-stage 0 +START 3165618: Tue 28 Feb 2023 12:00:00 AM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 46.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 34.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 45.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 48.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 47.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 42.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 42.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +5: Launching on nid005901 (5/8), master nid005386 port 9999, GPUs 8, CUDA: True +3: Launching on nid005529 (3/8), master nid005386 port 9999, GPUs 8, CUDA: True +2: Launching on nid005528 (2/8), master nid005386 port 9999, GPUs 8, CUDA: True +1: Launching on nid005387 (1/8), master nid005386 port 9999, GPUs 8, CUDA: True +4: Launching on nid005900 (4/8), master nid005386 port 9999, GPUs 8, CUDA: True +0: Launching on nid005386 (0/8), master nid005386 port 9999, GPUs 8, CUDA: True +7: Launching on nid006591 (7/8), master nid005386 port 9999, GPUs 8, CUDA: True +6: Launching on nid006590 (6/8), master nid005386 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/3165618.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... True +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m2b7100mval +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m2b7100m +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m2b7100m +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m2b7100mval +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-28 00:00:49,499] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +7: > setting tensorboard ... +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.099 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.cuda.o scaled_upper_triang_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_cuda.o layer_norm_hip_kernel.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 18.446 seconds +0: time to initialize megatron (seconds): 38.807 +0: [after megatron is initialized] datetime: 2023-02-28 00:01:10 +0: building GPT model ... +0: [2023-02-28 00:01:10,942] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-28 00:01:10,943] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-28 00:01:10,943] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.38 GB, percent = 6.0% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-28 00:01:12,935] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-28 00:01:13,167] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-28 00:01:13,167] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-28 00:01:13,168] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.42 GB, percent = 6.0% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-28 00:01:13,170] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-28 00:01:26,407] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-28 00:01:26,408] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-28 00:01:26,408] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-28 00:01:26,414] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-28 00:01:26,414] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-28 00:01:26,534] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-28 00:01:26,535] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-28 00:01:26,535] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.11 GB, percent = 6.2% +0: ninja: no work to do. +0: Time to load utils op: 0.20525360107421875 seconds +0: Time to load utils op: 0.10221076011657715 seconds +0: [2023-02-28 00:01:26,762] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-28 00:01:26,763] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-28 00:01:26,763] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.11 GB, percent = 6.2% +0: ninja: no work to do. +0: Time to load utils op: 0.13227534294128418 seconds +0: Time to load utils op: 0.0006759166717529297 seconds +0: Time to load utils op: 0.0004603862762451172 seconds +0: Time to load utils op: 0.20225214958190918 seconds +0: Time to load utils op: 0.20258808135986328 seconds +0: Time to load utils op: 0.2024521827697754 seconds +0: Time to load utils op: 0.20229601860046387 seconds +0: Time to load utils op: 0.20239663124084473 seconds +0: [2023-02-28 00:01:26,886] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-28 00:01:26,887] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-28 00:01:26,887] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.15 GB, percent = 6.2% +3: Time to load utils op: 0.21117091178894043 seconds +3: Time to load utils op: 0.2116708755493164 seconds +3: Time to load utils op: 0.21108126640319824 seconds +3: Time to load utils op: 0.21048307418823242 seconds +3: Time to load utils op: 0.21157097816467285 seconds +3: Time to load utils op: 0.21130752563476562 seconds +3: Time to load utils op: 0.21121454238891602 seconds +3: Time to load utils op: 0.21068930625915527 seconds +2: Time to load utils op: 0.21056866645812988 seconds +2: Time to load utils op: 0.21059918403625488 secondsTime to load utils op: 0.21059012413024902 seconds +2: +2: Time to load utils op: 0.21058917045593262 seconds +2: Time to load utils op: 0.21059513092041016 secondsTime to load utils op: 0.21059775352478027 secondsTime to load utils op: 0.2106032371520996 seconds +2: +2: +2: Time to load utils op: 0.21061062812805176 seconds +0: Time to load utils op: 0.00043463706970214844 seconds +0: Time to load utils op: 0.0004093647003173828 seconds +0: Time to load utils op: 0.0003833770751953125 seconds +0: Time to load utils op: 0.00038123130798339844 seconds +0: Time to load utils op: 0.0004425048828125 seconds +1: Time to load utils op: 0.24348044395446777 seconds +1: Time to load utils op: 0.24349403381347656 seconds +6: Time to load utils op: 0.24079465866088867 secondsTime to load utils op: 0.2409822940826416 seconds +6: +6: Time to load utils op: 0.24068713188171387 seconds +1: Time to load utils op: 0.24349594116210938 seconds +6: Time to load utils op: 0.23988032341003418 seconds +1: Time to load utils op: 0.243516206741333 secondsTime to load utils op: 0.24329829216003418 seconds +1: +6: Time to load utils op: 0.23719048500061035 seconds +1: Time to load utils op: 0.24285173416137695 seconds +6: Time to load utils op: 0.23967790603637695 seconds +6: Time to load utils op: 0.2382335662841797 secondsTime to load utils op: 0.23883891105651855 seconds +6: +1: Time to load utils op: 0.24357962608337402 secondsTime to load utils op: 0.24358296394348145 seconds +1: +4: Time to load utils op: 0.24128198623657227 secondsTime to load utils op: 0.24127483367919922 seconds +4: +7: Time to load utils op: 0.2393357753753662 secondsTime to load utils op: 0.23924684524536133 seconds +7: Time to load utils op: 0.23925018310546875 seconds +7: Time to load utils op: 0.23926210403442383 seconds +7: +4: Time to load utils op: 0.24129986763000488 seconds +4: Time to load utils op: 0.24136757850646973 seconds +4: Time to load utils op: 0.24136686325073242 secondsTime to load utils op: 0.2413177490234375 seconds +7: Time to load utils op: 0.23927736282348633 seconds +4: +7: Time to load utils op: 0.23928093910217285 secondsTime to load utils op: 0.2392873764038086 seconds +7: +7: Time to load utils op: 0.2392592430114746 seconds +4: Time to load utils op: 0.24137163162231445 seconds +4: Time to load utils op: 0.24132704734802246 seconds +5: Time to load utils op: 0.2399449348449707 seconds +5: Time to load utils op: 0.23996543884277344 seconds +5: Time to load utils op: 0.23998641967773438 seconds +5: Time to load utils op: 0.23999500274658203 secondsTime to load utils op: 0.2400059700012207 seconds +5: +5: Time to load utils op: 0.24003815650939941 secondsTime to load utils op: 0.2400035858154297 seconds +5: Time to load utils op: 0.24000787734985352 seconds +5: +3: Time to load utils op: 0.0008904933929443359 seconds +3: Time to load utils op: 0.0010747909545898438 secondsTime to load utils op: 0.0010640621185302734 seconds +3: +3: Time to load utils op: 0.0010464191436767578 seconds +3: Time to load utils op: 0.0011408329010009766 seconds +3: Time to load utils op: 0.0012073516845703125 seconds +3: Time to load utils op: 0.0011615753173828125 seconds +3: Time to load utils op: 0.0011734962463378906 seconds +2: Time to load utils op: 0.0009794235229492188 seconds +2: Time to load utils op: 0.0010406970977783203 secondsTime to load utils op: 0.0009970664978027344 seconds +2: +2: Time to load utils op: 0.0009958744049072266 seconds +2: Time to load utils op: 0.0009899139404296875 seconds +2: Time to load utils op: 0.0010209083557128906 seconds +2: Time to load utils op: 0.00102996826171875 seconds +2: Time to load utils op: 0.001094818115234375 seconds +4: Time to load utils op: 0.0008177757263183594 seconds +4: Time to load utils op: 0.0008504390716552734 seconds +4: Time to load utils op: 0.001035928726196289 seconds +4: Time to load utils op: 0.0010881423950195312 secondsTime to load utils op: 0.0010938644409179688 seconds +4: Time to load utils op: 0.0011277198791503906 seconds +4: +7: Time to load utils op: 0.0011322498321533203 seconds +4: Time to load utils op: 0.0010766983032226562 seconds +7: Time to load utils op: 0.0013082027435302734 seconds +4: Time to load utils op: 0.0010859966278076172 seconds +7: Time to load utils op: 0.0014224052429199219 secondsTime to load utils op: 0.0014350414276123047 seconds +7: +7: Time to load utils op: 0.0014295578002929688 secondsTime to load utils op: 0.0013151168823242188 seconds +7: +7: Time to load utils op: 0.001474142074584961 seconds +7: Time to load utils op: 0.0013456344604492188 seconds +1: Time to load utils op: 0.0010576248168945312 seconds +1: Time to load utils op: 0.0011627674102783203 seconds +1: Time to load utils op: 0.0012753009796142578 seconds +1: Time to load utils op: 0.0013856887817382812 seconds +1: Time to load utils op: 0.0014605522155761719 secondsTime to load utils op: 0.0014040470123291016 seconds +1: +1: Time to load utils op: 0.0013794898986816406 seconds +1: Time to load utils op: 0.0014564990997314453 seconds +6: Time to load utils op: 0.0005121231079101562 secondsTime to load utils op: 0.0005342960357666016 seconds +6: +6: Time to load utils op: 0.0005435943603515625 seconds +6: Time to load utils op: 0.00038242340087890625 seconds +6: Time to load utils op: 0.0005228519439697266 seconds +6: Time to load utils op: 0.0003914833068847656 seconds +6: Time to load utils op: 0.0003695487976074219 seconds +6: Time to load utils op: 0.0006105899810791016 seconds +5: Time to load utils op: 0.0010023117065429688 seconds +5: Time to load utils op: 0.0008099079132080078 seconds +5: Time to load utils op: 0.0012607574462890625 seconds +5: Time to load utils op: 0.0010721683502197266 seconds +5: Time to load utils op: 0.0012581348419189453 seconds +5: Time to load utils op: 0.0011217594146728516 seconds +5: Time to load utils op: 0.001238107681274414 seconds +5: Time to load utils op: 0.0011534690856933594 seconds +0: [2023-02-28 00:01:27,008] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-28 00:01:27,009] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-28 00:01:27,009] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,119] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-28 00:01:27,120] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,120] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,227] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-28 00:01:27,227] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,228] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,337] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-28 00:01:27,337] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,338] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,443] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-28 00:01:27,443] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,443] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,552] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-28 00:01:27,553] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,553] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,657] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-28 00:01:27,658] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 00:01:27,658] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: [2023-02-28 00:01:27,658] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-28 00:01:27,658] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-28 00:01:27,658] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-28 00:01:27,658] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-28 00:01:27,659] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-28 00:01:27,660] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-28 00:01:27,661] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-28 00:01:27,661] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.00042319297790527344 seconds +0: [2023-02-28 00:01:27,661] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-28 00:01:27,739] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-28 00:01:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-28 00:01:28,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-28 00:01:28,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-28 00:01:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-28 00:01:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-28 00:01:28,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-28 00:01:28,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-28 00:01:28,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-28 00:01:28,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-28 00:01:28,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-28 00:01:28,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-28 00:01:28,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-28 00:01:28,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-28 00:01:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-28 00:01:28,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-28 00:01:28,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-28 00:01:28,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-28 00:01:28,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-28 00:01:28,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-28 00:01:28,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-28 00:01:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-28 00:01:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-28 00:01:28,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-28 00:01:28,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-28 00:01:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-28 00:01:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-28 00:01:28,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-28 00:01:28,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-28 00:01:28,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-28 00:01:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-28 00:01:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-28 00:01:28,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-28 00:01:28,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-28 00:01:28,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-28 00:01:28,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-28 00:01:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-28 00:01:28,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-28 00:01:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-28 00:01:28,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-28 00:01:28,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:28,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-28 00:01:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:28,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:28,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:28,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-28 00:01:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-28 00:01:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:28,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:28,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:28,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:28,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-28 00:01:29,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-28 00:01:29,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:29,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:29,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:29,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:29,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:29,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:29,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:29,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-28 00:01:29,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-28 00:01:29,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-28 00:01:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-28 00:01:29,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-28 00:01:29,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-28 00:01:29,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-28 00:01:29,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-28 00:01:29,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-28 00:01:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-28 00:01:29,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-28 00:01:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-28 00:01:29,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-28 00:01:29,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-28 00:01:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-28 00:01:29,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-28 00:01:29,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-28 00:01:29,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-28 00:01:29,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-28 00:01:29,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-28 00:01:29,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-28 00:01:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-28 00:01:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-28 00:01:29,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-28 00:01:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-28 00:01:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-28 00:01:29,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-28 00:01:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-28 00:01:29,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-28 00:01:29,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-28 00:01:29,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-28 00:01:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-28 00:01:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-28 00:01:29,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-28 00:01:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-28 00:01:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-28 00:01:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-28 00:01:29,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-28 00:01:29,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-28 00:01:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-28 00:01:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-28 00:01:29,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-28 00:01:29,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-28 00:01:29,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-28 00:01:29,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-28 00:01:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-28 00:01:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-28 00:01:29,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-28 00:01:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-28 00:01:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-28 00:01:29,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-28 00:01:29,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-28 00:01:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-28 00:01:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-28 00:01:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-28 00:01:29,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-28 00:01:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-28 00:01:29,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-28 00:01:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-28 00:01:29,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:29,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-28 00:01:29,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:29,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-28 00:01:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-28 00:01:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:29,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:30,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:30,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:30,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-28 00:01:30,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-28 00:01:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-28 00:01:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-28 00:01:30,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:30,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-28 00:01:30,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-28 00:01:30,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-28 00:01:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-28 00:01:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-28 00:01:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-28 00:01:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-28 00:01:30,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-28 00:01:30,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-28 00:01:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-28 00:01:30,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-28 00:01:30,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-28 00:01:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-28 00:01:30,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-28 00:01:30,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-28 00:01:30,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-28 00:01:30,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-28 00:01:30,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-28 00:01:30,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-28 00:01:30,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-28 00:01:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-28 00:01:30,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-28 00:01:30,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-28 00:01:30,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-28 00:01:30,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-28 00:01:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-28 00:01:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-28 00:01:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:30,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-28 00:01:30,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-28 00:01:30,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-28 00:01:30,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-28 00:01:30,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-28 00:01:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:30,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-28 00:01:30,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-28 00:01:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-28 00:01:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:31,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-28 00:01:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-28 00:01:31,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-28 00:01:31,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-28 00:01:31,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-28 00:01:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-28 00:01:31,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-28 00:01:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-28 00:01:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-28 00:01:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-28 00:01:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-28 00:01:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-28 00:01:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-28 00:01:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-28 00:01:31,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-28 00:01:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-28 00:01:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-28 00:01:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-28 00:01:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-28 00:01:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-28 00:01:31,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-28 00:01:31,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-28 00:01:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-28 00:01:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-28 00:01:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-28 00:01:31,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-28 00:01:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-28 00:01:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-28 00:01:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-28 00:01:31,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-28 00:01:31,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-28 00:01:31,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-28 00:01:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-28 00:01:31,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-28 00:01:31,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-28 00:01:31,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-28 00:01:31,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-28 00:01:31,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-28 00:01:31,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-28 00:01:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-28 00:01:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-28 00:01:31,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-28 00:01:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-28 00:01:31,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-28 00:01:31,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-28 00:01:31,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-28 00:01:31,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-28 00:01:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-28 00:01:31,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-28 00:01:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-28 00:01:31,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-28 00:01:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-28 00:01:31,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-28 00:01:31,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-28 00:01:31,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-28 00:01:31,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-28 00:01:31,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-28 00:01:31,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-28 00:01:31,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-28 00:01:31,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-28 00:01:31,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-28 00:01:31,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-28 00:01:31,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-28 00:01:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-28 00:01:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-28 00:01:31,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +6: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-28 00:01:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-28 00:01:31,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-28 00:01:31,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-28 00:01:31,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +4: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-28 00:01:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-28 00:01:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-28 00:01:31,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-28 00:01:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-28 00:01:31,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-28 00:01:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-28 00:01:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +7: [2023-02-28 00:01:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,224] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +7: [2023-02-28 00:01:32,228] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +6: [2023-02-28 00:01:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +6: [2023-02-28 00:01:32,249] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +7: [2023-02-28 00:01:32,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +1: [2023-02-28 00:01:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,277] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +7: [2023-02-28 00:01:32,277] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +1: [2023-02-28 00:01:32,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +4: [2023-02-28 00:01:32,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,286] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +4: [2023-02-28 00:01:32,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,289] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +4: [2023-02-28 00:01:32,290] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +5: [2023-02-28 00:01:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +4: [2023-02-28 00:01:32,293] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +3: [2023-02-28 00:01:32,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,294] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +5: [2023-02-28 00:01:32,295] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +3: [2023-02-28 00:01:32,298] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +6: [2023-02-28 00:01:32,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,310] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +5: [2023-02-28 00:01:32,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,311] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +1: [2023-02-28 00:01:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,313] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +6: [2023-02-28 00:01:32,314] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +5: [2023-02-28 00:01:32,315] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +1: [2023-02-28 00:01:32,317] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +2: [2023-02-28 00:01:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,326] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-28 00:01:32,330] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +5: [2023-02-28 00:01:32,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,331] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +5: [2023-02-28 00:01:32,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,332] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-02-28 00:01:32,335] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +5: [2023-02-28 00:01:32,336] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +6: [2023-02-28 00:01:32,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +3: [2023-02-28 00:01:32,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +7: [2023-02-28 00:01:32,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +6: [2023-02-28 00:01:32,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +6: [2023-02-28 00:01:32,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +3: [2023-02-28 00:01:32,346] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +7: [2023-02-28 00:01:32,347] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +6: [2023-02-28 00:01:32,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +2: [2023-02-28 00:01:32,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,349] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +0: [2023-02-28 00:01:32,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,350] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +2: [2023-02-28 00:01:32,353] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +0: [2023-02-28 00:01:32,354] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +2: [2023-02-28 00:01:32,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,354] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +6: [2023-02-28 00:01:32,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,356] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +2: [2023-02-28 00:01:32,358] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +1: [2023-02-28 00:01:32,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,359] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +6: [2023-02-28 00:01:32,360] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +4: [2023-02-28 00:01:32,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,360] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +2: [2023-02-28 00:01:32,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,360] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +1: [2023-02-28 00:01:32,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +4: [2023-02-28 00:01:32,364] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +2: [2023-02-28 00:01:32,364] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +6: [2023-02-28 00:01:32,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,372] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +5: [2023-02-28 00:01:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +7: [2023-02-28 00:01:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +7: [2023-02-28 00:01:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +5: [2023-02-28 00:01:32,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +4: [2023-02-28 00:01:32,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,375] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +3: [2023-02-28 00:01:32,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,376] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +6: [2023-02-28 00:01:32,376] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +5: [2023-02-28 00:01:32,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-28 00:01:32,377] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +5: [2023-02-28 00:01:32,377] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +7: [2023-02-28 00:01:32,378] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +5: [2023-02-28 00:01:32,378] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +0: [2023-02-28 00:01:32,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +0: [2023-02-28 00:01:32,379] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +3: [2023-02-28 00:01:32,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +4: [2023-02-28 00:01:32,380] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +5: [2023-02-28 00:01:32,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +0: [2023-02-28 00:01:32,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,382] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-28 00:01:32,383] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +2: [2023-02-28 00:01:32,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +0: could not find arguments in the checkpoint ... +2: [2023-02-28 00:01:32,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +0: checkpoint version 3.0 +0: [2023-02-28 00:01:32,386] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +4: [2023-02-28 00:01:32,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,387] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +2: [2023-02-28 00:01:32,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,387] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +2: [2023-02-28 00:01:32,389] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +3: [2023-02-28 00:01:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,391] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +1: [2023-02-28 00:01:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,392] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +4: [2023-02-28 00:01:32,392] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +2: [2023-02-28 00:01:32,392] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +3: [2023-02-28 00:01:32,395] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +1: [2023-02-28 00:01:32,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +2: [2023-02-28 00:01:32,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,398] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +7: [2023-02-28 00:01:32,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,398] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +1: [2023-02-28 00:01:32,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,399] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +2: [2023-02-28 00:01:32,402] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +5: [2023-02-28 00:01:32,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,402] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +5: [2023-02-28 00:01:32,402] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +1: [2023-02-28 00:01:32,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +6: [2023-02-28 00:01:32,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,405] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +5: [2023-02-28 00:01:32,406] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +4: [2023-02-28 00:01:32,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +3: [2023-02-28 00:01:32,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +6: [2023-02-28 00:01:32,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +4: [2023-02-28 00:01:32,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,411] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +4: [2023-02-28 00:01:32,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-28 00:01:32,411] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +4: [2023-02-28 00:01:32,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +3: [2023-02-28 00:01:32,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +6: [2023-02-28 00:01:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-28 00:01:32,414] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +4: [2023-02-28 00:01:32,415] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +4: [2023-02-28 00:01:32,416] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +6: [2023-02-28 00:01:32,419] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +3: [2023-02-28 00:01:32,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +7: [2023-02-28 00:01:32,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +7: [2023-02-28 00:01:32,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-28 00:01:32,426] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +3: [2023-02-28 00:01:32,426] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +7: [2023-02-28 00:01:32,429] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +0: [2023-02-28 00:01:32,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,430] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +7: [2023-02-28 00:01:32,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +0: [2023-02-28 00:01:32,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,432] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +0: [2023-02-28 00:01:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,433] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +1: [2023-02-28 00:01:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,433] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +0: [2023-02-28 00:01:32,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:32,434] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +0: [2023-02-28 00:01:32,435] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +0: [2023-02-28 00:01:32,436] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +1: [2023-02-28 00:01:32,437] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +0: [2023-02-28 00:01:32,437] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +0: [2023-02-28 00:01:32,440] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +3: [2023-02-28 00:01:32,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:32,441] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +1: [2023-02-28 00:01:32,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,441] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +2: [2023-02-28 00:01:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-28 00:01:32,442] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +3: [2023-02-28 00:01:32,445] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +1: [2023-02-28 00:01:32,445] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +2: [2023-02-28 00:01:32,447] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +1: [2023-02-28 00:01:32,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-28 00:01:32,484] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +1: [2023-02-28 00:01:32,489] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +0: [2023-02-28 00:01:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-28 00:01:33,450] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +0: [2023-02-28 00:01:33,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +3: [2023-02-28 00:01:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b7100m/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-28 00:01:33,485] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +3: [2023-02-28 00:01:33,488] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +0: successfully loaded checkpoint from checkpoints_619m2b7100m at iteration 0 +7: time (ms) | load-checkpoint: 5756.91 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-28 00:01:33 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.006697 seconds +0: number of documents: 208931 +0: > dataset split: +0: train: +0: document indices in [0, 208931) total of 208931 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.051 seconds +0: total number of samples: 48805 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.045998 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.061 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-28 00:01:46 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 22889.32 | train/valid/test-data-iterators-setup: 12178.85 +0: [after training is done] datetime: 2023-02-28 00:01:46 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 3.829336E+00 | lm loss PPL: 4.603196E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 3165618: Tue 28 Feb 2023 12:02:16 AM EET diff --git a/619m2b7100m/sbatch_619m2b7100m.sh b/619m2b7100m/sbatch_619m2b7100m.sh new file mode 100644 index 0000000000000000000000000000000000000000..70bf0ceab4aaed324310e1e19d2bb552cdf9e438 --- /dev/null +++ b/619m2b7100m/sbatch_619m2b7100m.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b7100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1_308_594 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 13_086 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b7100m/sbatch_619m2b7100mval.sh b/619m2b7100m/sbatch_619m2b7100mval.sh new file mode 100644 index 0000000000000000000000000000000000000000..de2c7c520828ca5eb722fcbe41fb958450626a0e --- /dev/null +++ b/619m2b7100m/sbatch_619m2b7100mval.sh @@ -0,0 +1,170 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b7100mval +VARIANT_CKPT=619m2b7100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT +# Start from scratch +#rm -rf "$CHECKPOINT_PATH" "$TENSORBOARD_PATH" + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --no-load-optim \ + --reset-progress \ + --override-lr-scheduler \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b7100m/tensorboard_619m2b7100m/events.out.tfevents.1677502283.nid006608.12371.0 b/619m2b7100m/tensorboard_619m2b7100m/events.out.tfevents.1677502283.nid006608.12371.0 new file mode 100644 index 0000000000000000000000000000000000000000..ff5e0d03f280c7440d9e032f033934578bb28a6b --- /dev/null +++ b/619m2b7100m/tensorboard_619m2b7100m/events.out.tfevents.1677502283.nid006608.12371.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aecded72f0ece05f81ff539904e6ef5a5e7e333914ab2e2da5943a8248fb0ad8 +size 9099912 diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..63d2389603e22e1b6d2ddf311b96c32b5244cd09 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.24826051946595584, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.022223944257084786}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.058163199053238857, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012927818826889883}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2464607565329814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004099130993659631}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08900765115929998, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017737031911390954}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.027958228185354133, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008197899214780566}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1212206967206692, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002879981343925423}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04299602830782456, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011529371606547893}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05687669768133077, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012334222658519987}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.24263081943381815, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004049339934122269}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.08719970330894132, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017067065975610177}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05627985268944361, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012320876299857706}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.23899011890339064, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003907803465727549}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.08616972905837225, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016929629302388789}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ed5da020790491b9eb277b8d537e7848258e4946 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.17248776305808664, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.016795777205216727}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05236545537858825, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001386009142665529}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.23952149357583752, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003964748870758093}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07974711563423877, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001617468016734832}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.022770616206251718, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007222294749316485}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.10196623196520473, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002694349434124461}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03518570200418623, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001033184512581815}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05119901012081213, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013530554709038835}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.23448910434022022, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003865316164374324}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07801688157847861, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015709474468090805}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.050684197186769614, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013658718701688602}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2299271822021421, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003790698171203416}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07704409187166199, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001581413920310377}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0fdd93570f89cf8d3d689a84c47b3c902bdf2ed1 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1740921740768521, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02245852136332944}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05092812801458869, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011757535036755983}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.23029941909180796, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003907134419847019}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.0783917476843809, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016004491851345573}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02209321030425811, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007051200067221665}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.09747850658202363, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0025868322745989537}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03402987121771126, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010090793783559}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.049405325142194104, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011230235282996012}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.22350619198052418, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0037728669875094984}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07616316741043358, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015474107907167969}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.048969182406106454, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011392876466907232}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.22012323480721052, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036888555418374417}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0753146662045205, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001548681630951826}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a074cead511d9fa42dfcc9149219eccd8f99e --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1714945768037504, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01842253558598964}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.051722195443120254, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012608405872136094}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.23090502211754627, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003988254256945192}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07920093236705308, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015997822971242884}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.022332629611478836, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006936318961594426}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.09917949581608851, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0026296436063060185}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03455130562554136, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010017255576368568}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.050294237259899566, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012321105827659488}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2246039480148206, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003881559343048008}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07701796251510407, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015576337881804197}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.049636917580444036, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012252524302076237}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2198310121914411, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037347619995472055}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07587583163215018, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015390342361881646}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..dadf08f606be231450d506034c305293a6afac7d --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.2034962070733705, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02512817739445294}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.051358266508139044, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011609551790621932}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2308574801783086, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004058824692805349}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07919122114553223, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015848708191778377}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.022201219787716792, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006921390509979684}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1003506898317558, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0026531796117823313}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03439575337322024, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009973225938294027}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0497139845045926, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011125764616461975}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2236437131770874, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003919177303008036}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07671934522499119, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015289968779697359}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.04929778407737348, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001109857222017746}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2206395869777806, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037909884211304277}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07599501751355653, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015187997824729323}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..3eca928b878e2a0035d81c202e4061d6f6c49a2d --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.19761112575452508, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.019235465539383146}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.052579153244265424, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011921637714878187}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.23535879092052564, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0041071634335935175}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08085878644777693, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016289906236916856}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.022671150422158365, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000709183367358532}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.10160646453537654, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002707170710882478}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03502968692047543, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010231866444401123}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.050647877113144134, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001139117442858445}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2275596229219294, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003973700969182821}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07799816636434088, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015628913834455332}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.050414491464882374, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011507171716506196}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.22494103941116889, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0038872663249574415}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07750080873328469, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001570637280878507}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d42acc7d8a10729705345732aa8214aa599eaf12 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.07334286701079117, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014209616553442557}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.11781728898648271, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002048222920943715}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.08341229365529593, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001426857348540837}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.007614750236728455, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003759989992852565}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.013390248256536815, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000802825111743644}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.00877170766892616, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004188649332932362}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.06575808443610535, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011792063673595292}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.10763379445866503, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018034771592472351}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.07535689248472442, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012035067304151934}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.06788713608018786, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013039873220152538}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.10982226932159775, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001904635097262412}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.07735895099517735, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013105886577076619}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.45549657362697604, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04494579402966035}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2c00e575279480c355c2834b7e67bddb75f8ff11 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.09377324802061514, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014183788800124697}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.14181618673418064, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001999709640963563}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.10367919976235754, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013958183292936235}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.007148933952844262, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003561943868556437}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.012216117762769886, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007004781288235548}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.008218722219301039, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000410740325223334}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07479319586763088, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010178455763075111}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.1156679097936295, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015562821873263057}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08313407829979999, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009900147300689925}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.08828296890875521, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013091560120064637}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.134369194217566, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018930565890544265}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09773392872756465, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001288712610683545}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4342251027303836, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04156452799201429}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8cec0dec173b65e96072b5af436c9670e635e3be --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.09094561531157444, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001456157079862017}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.12714249099405814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018044241583523653}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.0960491519485715, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001321339124082858}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006679127466354158, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004080467799025574}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.009968832268195739, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006285558582734068}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.007144914805869838, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00039460890013464246}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0766627034344775, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011486123351465115}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.10991861742729633, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001524234817970641}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0815527888639139, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010329480675385994}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.08548630034360417, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001361329490189725}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11951122406072837, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001667021801162019}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09016504132142787, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001216230722231443}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4311595587295042, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05311201104959919}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7377faecfd4d50b049a0a939629cdad13034a0d4 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08508624990590842, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018605756778959563}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.11062172170633029, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020249317253040237}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.08383654969005261, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014652925304525217}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.008492614408520994, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005516284768616702}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.011602511796966591, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006906008777318128}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.008305764810262975, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00044353230048316586}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07393459992048995, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015722641471155711}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.09842577979303434, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017872350704345923}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0733905772354038, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012191555989918234}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07932181599326994, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017467205552871912}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.10333223732938228, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018867954598698327}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.07804802875444604, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013485565230468167}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4945744591124352, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04112490496696115}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0a981cb6d4ac7ce71640abe87f3958a157b63874 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.029004586891320194, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001352875160534484}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.03766653675736129, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0015747667565044058}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.02787863882909451, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0011476158555931272}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0032795699585914644, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00040022837086021713}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.004358266731620199, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004899708073278473}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0030035258951381554, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00029734523782658787}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.025907175815949448, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011777866749689255}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.034451613541809104, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001433290325245525}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.025080472403581883, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010020364992695604}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.026962480466237466, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012715176574721234}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.03489216949042506, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0014614184198085247}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.025723161174858334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001051539558458842}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.1530683867528085, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02304865388359797}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..50ebce3fc6cf6bb6cee5793ad137b7daf0696469 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.004746445530663895, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0005682413591016014}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.005894862050119441, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006913140760157219}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.004344032704921915, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00048769273143641834}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.00044307036215439545, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0001004207530010917}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0006567436597417495, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0001789950872715256}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.00045459377689834523, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00010522677369073278}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.004375980531133334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005213355452750296}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.005438941343166284, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.000635980379876231}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.00397221141689711, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00043801211935723714}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.004469825621096149, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005385326208085657}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.005512464579518613, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0006502051982674122}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.004059385391361045, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0004536081698576505}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.6420481653021324e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 8.440587103346813e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ddfb65691c4f19f9a401a3c9c1c49a69a9dd06b1 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 0.5913211031265735, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03714947332384052}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.08058873419557824, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0032571051434779103}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.06170899966771193, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020401649662685367}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.057779110162827135, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018983542761005778}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.02581229791743323, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015932358672241216}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.015789878220228096, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007208101441802875}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.01574069846670012, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007666963173557723}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.06923344159739564, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0027564694592522388}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.05708601514024535, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019396856322431723}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.05161308138091674, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016887665841841223}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.06476318617469784, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0027058888434037384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.04796277405126694, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0015865538487918089}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.04557718964996215, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015307841093839293}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..72e7ebdefbb834619aabbfbe3944313bf9a9c00b --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.7129526592901363, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12569632286144403}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.33757710589590273, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0027406590712350525}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.27297090245543504, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023001976057750945}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.2847694417389595, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002040695495961792}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.08861232666101841, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001955054629766258}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.0692950974716365, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013251834733069956}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.07230524386133462, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013218286978928981}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.24807522899572165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0022872616728098504}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.19802854633734265, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017557148314317694}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2070591630578462, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015735715497195613}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.28009204868111853, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025289533444105197}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.22495897081460098, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002023117019898989}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.23499367959584552, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018330473958537736}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..746901cc63e346c5f23b3ee423c7dd84e9583cc7 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.462491951458752, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12932178148201628}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3164123369697131, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0034775181138680792}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.2598745794315416, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027077821224868893}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.2636634022685503, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0024633437175046883}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.09371196451152501, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0027593269396471646}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.06611332250980292, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013564334736745416}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06739797982186209, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001304257328973498}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.23561396433786047, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003025416778360062}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.1885327325268679, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020407871024373837}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.19174609701033776, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018513547565069343}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2651518207988083, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003239070270009198}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.2148834164583239, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023603423697314426}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.21828405633341527, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002159399848115338}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9900c996fc736a1fe837ec7f85379400bfee11bd --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.1741335461954034, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.17453804662011688}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.29613934206804304, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004451198145421545}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.22402208739224386, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030604385370891414}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.22501699083270982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002791442670232082}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.11347371004313266, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00401773931870467}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.06026738126786902, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014084605676934583}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06090634543978955, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013142776626892116}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.231748269398982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.004130558382055014}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.1649071700002769, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022872947567682198}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.16691558304091286, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020999665649599486}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2556362993648324, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00426628185731524}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.1872697901304377, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002639460546137773}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.1886311008196838, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024055139053464868}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..d509ce74f10a5687e9bfc82ffdc20dc4f1c3c6cd --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.5346862761802016, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12479053039126499}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.31262165964896604, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.005047999697934806}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1998465379109657, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030245662869133787}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.21168945879136675, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0028856185357249834}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.1388925348990009, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.004790445901510994}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.05572347503457502, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013458562601325785}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06017636185529871, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013291090944976105}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.253696212330601, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.004793613646337102}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.1499151276496759, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002283351691483937}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.16035037300782284, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0021851407273096933}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.27512771480546755, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004896008952662329}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.167957116845922, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002583473884701478}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.1790062741536553, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002475640029677632}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..d79613eb2dbf09efdcb81e0045510f169a465140 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.4029523973418088, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10662120310428724}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3260744222268218, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.005383991884322823}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.19321564713578962, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030459526877180125}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.20707330180497546, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0029170286213066506}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.15881987477547233, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.005245943651586924}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.05630141761843497, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013558505278503214}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06161675662277744, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013345748600508244}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2699334173910297, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.005178522168977427}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.14605882274029522, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022822110698795745}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.1584188601757907, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002197211198869285}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.29011330437239946, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.005265135503554526}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.16316644716956302, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026017608845607804}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.17603000847259365, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0025014903430049438}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_0.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..27e8604cbeb26d9ae01ed50a6840dd4ad5b0e67c --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09522462576051435, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015560425280753526}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.23077643089582087, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0034938489740306655}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.13237124936382566, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002030318110427792}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.014595206517583913, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006912221325529861}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03697330908984525, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017262156912820615}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.020570971802076836, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009528220708571039}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.08394414353545941, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012782778620016224}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2050805363011997, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002985550618482208}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11697861341242026, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00168201405286699}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0729362594061639, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001222373131462693}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.17884876600176577, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002908606459113646}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1016392539142757, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016105438516145294}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.7376401152029481, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0628176626129689}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_1.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..021a9fdff5af31d8fe396276a4aaa6516363f887 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08993167019157738, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014994262109755236}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2202950978484778, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003503324448762303}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.1261090141950385, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002038712200394186}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011719494218269198, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006320218059713919}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02990361241325009, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016266190349500753}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01664837239561062, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008965007944526279}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07897756505863603, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001243259199520003}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.19438615998458816, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029533459158402457}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11086188237596728, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016892809835885039}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06886845396607581, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011593284126768632}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1706924968758996, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028400656934941242}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0968810722814496, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015962633597926558}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6702693938939304, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07803616330676291}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_2.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..04303efa959ac8c54bf4d43afb78ee25d7a6fc78 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08782405624093549, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014766059696371997}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2160456627932237, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003441332688585382}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12317572309436386, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019948599460197793}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.012333383775107388, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006521057452803417}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03179899644401759, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001722233233752029}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.017543316635811195, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009239971756884746}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07776394031041892, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012329815052524796}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.19243820960438146, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029224083235941836}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.1092412067078929, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016683921666508167}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0676638257670816, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011247421201455816}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.16808542682817457, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027515286144884723}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09510788828477747, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015320545647342031}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6510687810492491, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09062967968019935}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_3.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ac3024fd2610049ece018dad932c5da45bdd15bb --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08897486239114341, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017009074998627915}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2091191397530532, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003614643324036501}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12144582095736131, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002114335303171797}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.01259505859479755, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00069046073565253}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03122412533566738, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001678680477457119}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01754230953552776, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009355858622010496}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07838071014182374, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001396156009643834}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18573352825783288, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003039958738646297}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10734285670078089, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001745478308264722}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06823456301932596, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013342812919056666}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.16149631288933364, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028294650899024533}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0931643849311634, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016188390418371856}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.694494856973092, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09315806446376278}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_4.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..968f4a532e17ff00d4a8b3cd40391453ba29ee26 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.03203907131106538, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023266110566352526}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.05290796868421392, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030930646890496914}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.03483851864300909, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020507945477994454}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.005078726019956826, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008900002887502463}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.008305472613043527, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009638105580648599}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.005528078242750013, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007181419240142681}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.02793525308488722, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020306661448374493}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04674043181938062, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002691339349960188}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.03034968255806866, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017240783735724304}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.02589849825305961, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0020040582236913592}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.04190637777442485, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024786007167556857}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.027467434064520325, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016200826802563308}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.4634958897519297, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1078601254660865}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_5.json b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..820a4081b9173d88722771e1055b0742580046e7 --- /dev/null +++ b/619m2b71b5/evaluation/generation/agg.619m2b71b5_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.002739129716464841, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007581348910112568}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0023745897883191555, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006352252378761389}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0024866753710265106, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006684639506190258}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00047501133100467176, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00021640271325881454}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0003994503753543417, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0002011697037898624}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0004287346762241937, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002053885706784475}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.002126911972697357, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005869633815800918}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0018836680062018954, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005134831663617108}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0019488148577111895, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.000526299388991192}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0022796750730625366, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006491389537922883}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.001987887384982107, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005467706089350191}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0020723992306803042, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005716554317274563}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 9.409589204950959e-38, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.250977748694596e-32}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6605f36a8c7b002f4227a665ba9212ed69b0120c --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693759ead619f0c3d4d10bcdb01f8eef7599a6dc3e629c88524283b4e3634418 +size 4124855 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..df01ca019b0c5ef2ba28534d5a775f19c3b94021 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:868cd4008bd76a45bd69746d7608e4b2dede920e4c4153ec452dbd908707137a +size 5141711 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f92ef1dc60800e9274956464130d42a77214d34d --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926fd14251bd702ffbcdee0cf096dba3300106eae5fb7d0e0b4b00750000fc5f +size 5989931 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d50a1fed360a5bd28f7f8e7ac24cdf10bb0b59f2 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d1cad2b23a49b34aa0870a8aafe100edf733522ed5dc9da199f99407710f02 +size 6858935 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f6e410d334372d66d6657a65b681e0c5c8320fea --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a42888ca17b3a283e883fedaa75b82e017e631c0972323c417de736b4d5892d1 +size 7745057 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..aec8c4b670b76d83cbe0102d94af1554ba1870b1 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1323f7a19526272a1317315ba0bdf0e6a38f09062749bffe64c89bea6a22354b +size 8652985 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e7ed44a0dc21903bae88dc1e52b84c1c367df2e1 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb77e5560ea7331c6f10af9ca1d6cd2d3bdeba5774c0950805c42ef464fd6e53 +size 7619997 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..069b033dc38cfc53ffc5855abb01a81f1a35351b --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c458ec03ebc7d21c53a7977a5e5ac0aa45b1edd6881e5ff26d7c30031c3e63 +size 13322867 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8063919efd01bbd6598239bcc3b140d69dfbe6d8 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc389397a01f7d333f5df301e6ab636a67d7dd1c184d862ecd4f8107dd076763 +size 18900877 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d6ba801159fdd1316df817784c004d08a98cc09e --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c903aa36307c5dbc690dad3fc258002c034f862ca7bcc0b61062bd94feacf09c +size 24321948 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3c397aad23222a7061eedc48f6c29d27563960c9 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7a21536fc0499bbf5b616b61216e51956cc3ed98e5c23a739500b088648aaa5 +size 29471615 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..14cda7ed73e979b936b57d3d81f5b0c36e2c0ff6 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9df04393d70f3d3a3e1c5fda6e902582e998582e5438d7c70984888968b174fa +size 34798666 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5107c7f6ac62a5ab6c8ab450af9ce9155929b3f5 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0547402fc7fad0636cb7956ca156d3ae78465be056332e54e6e3b8bbad638c9 +size 3773286 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..266c8dfb2a826f715c3434d4d94ce05ae69eab76 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8df061c97bd83e3a1e4708946d5cc771b6e6b18d7c0094e7e61ee2b4e7bbfc07 +size 5040134 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4fb12242d9de7ae76480d9398371c809fe749a6f --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c305782e0daab4ec4ad4f1cf0082c7322f95fc654a7d39f0ed08a041f59e8f62 +size 6120902 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b3915d713d28272edf276ba21b32034624f46c87 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f1c07c4ab6b884e90ead1a39c723d2b31dc2c1081e83a1abc152401ca844509 +size 7162918 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b1b6051b061fa5b75d3eca415c296ee5833bc979 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883a5892f68cdbed55782fdbde5deafa5254f6b06b305c74c02bf710ab888b5b +size 8175419 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..64585d67d85009694289c42607cfb207752ef26c --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6078a411e9f093983251024a216595546b5114854dedfb6fe04fdf1a82b9f72 +size 9248521 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_0.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5cf474f8c118468ce0de564b467545e267af7c92 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f4909044df426105c6537ffd3888c8286d701c7467b08c427cd228d9590e33d +size 2833487 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_1.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3a8bae9ad085cfa1ca8c8829be159b6c9af172d9 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7142d5817a505320873985fda6a0d787fe732b3a8e71038e9a7f45a855a1b69d +size 5104500 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_2.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c1a14e173a363aec495bdfee903b52e75b6aeebb --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c1f58c139ac126c6eec4f2ed68575a6b34db0da773611a108324454fcdc7247 +size 7378206 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_3.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b4afd9934dfb41a8aaafaa35ce95244a123058e1 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e3150f1193b62409d8cc26c763d10f52f57e714034e8e67f6ad226c463f4048 +size 9647336 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_4.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a8409e0a719a6355c95a4535bdf18e1cc30b76af --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c626c86230336c5818b860c63f94c37489500c1c6c546f0a7fcb51af5bbe7109 +size 11672625 diff --git a/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_5.jsonl b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..23d53b98a1cacbeb2211f79b8e755d1e5f74e676 --- /dev/null +++ b/619m2b71b5/evaluation/generation/examples.619m2b71b5_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13accad32ecbc095cb0e340cf07f3b264e2fa02571c8395bf7f8e7f33aa480d0 +size 13897572 diff --git a/619m2b71b5/evaluation/generation/merged.csv b/619m2b71b5/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..663b1513f251e4ae510ea616c0eb2df5f1e57e33 --- /dev/null +++ b/619m2b71b5/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.01574069846670012 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.01574069846670012 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.07230524386133462 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.07230524386133462 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.06739797982186209 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.06739797982186209 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.06090634543978955 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.06090634543978955 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.06017636185529871 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.06017636185529871 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.06161675662277744 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.06161675662277744 +e2e_nlg_cleaned,5,average,multiple,0.05635723101129376 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.020570971802076836 +gem_xsum,0,median,rouge2_fmeasure,0.020570971802076836 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.01664837239561062 +gem_xsum,1,median,rouge2_fmeasure,0.01664837239561062 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.017543316635811195 +gem_xsum,2,median,rouge2_fmeasure,0.017543316635811195 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.01754230953552776 +gem_xsum,3,median,rouge2_fmeasure,0.01754230953552776 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.005528078242750013 +gem_xsum,4,median,rouge2_fmeasure,0.005528078242750013 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0004287346762241937 +gem_xsum,5,median,rouge2_fmeasure,0.0004287346762241937 +gem_xsum,5,average,multiple,0.013043630548000102 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.04299602830782456 +web_nlg_en,0,median,rouge2_fmeasure,0.04299602830782456 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.03518570200418623 +web_nlg_en,1,median,rouge2_fmeasure,0.03518570200418623 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.03402987121771126 +web_nlg_en,2,median,rouge2_fmeasure,0.03402987121771126 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.03455130562554136 +web_nlg_en,3,median,rouge2_fmeasure,0.03455130562554136 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.03439575337322024 +web_nlg_en,4,median,rouge2_fmeasure,0.03439575337322024 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.03502968692047543 +web_nlg_en,5,median,rouge2_fmeasure,0.03502968692047543 +web_nlg_en,5,average,multiple,0.03603139124149318 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.00877170766892616 +wiki_lingua_en,0,median,rouge2_fmeasure,0.00877170766892616 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.008218722219301039 +wiki_lingua_en,1,median,rouge2_fmeasure,0.008218722219301039 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.007144914805869838 +wiki_lingua_en,2,median,rouge2_fmeasure,0.007144914805869838 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.008305764810262975 +wiki_lingua_en,3,median,rouge2_fmeasure,0.008305764810262975 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.0030035258951381554 +wiki_lingua_en,4,median,rouge2_fmeasure,0.0030035258951381554 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.00045459377689834523 +wiki_lingua_en,5,median,rouge2_fmeasure,0.00045459377689834523 +wiki_lingua_en,5,average,multiple,0.005983204862732752 diff --git a/619m2b71b5/evaluation/generation/merged.json b/619m2b71b5/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..291b712d6f7d56d37dc35d21ee588825a45dc79d --- /dev/null +++ b/619m2b71b5/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.24826051946595584, "bleu_stderr": 0.022223944257084786, "rouge1_fmeasure": 0.08900765115929998, "rouge1_fmeasure_stderr": 0.0017737031911390954, "rouge1_precision": 0.058163199053238857, "rouge1_precision_stderr": 0.0012927818826889883, "rouge1_recall": 0.2464607565329814, "rouge1_recall_stderr": 0.004099130993659631, "rouge2_fmeasure": 0.04299602830782456, "rouge2_fmeasure_stderr": 0.0011529371606547893, "rouge2_precision": 0.027958228185354133, "rouge2_precision_stderr": 0.0008197899214780566, "rouge2_recall": 0.1212206967206692, "rouge2_recall_stderr": 0.002879981343925423, "rougeL_fmeasure": 0.08719970330894132, "rougeL_fmeasure_stderr": 0.0017067065975610177, "rougeL_precision": 0.05687669768133077, "rougeL_precision_stderr": 0.0012334222658519987, "rougeL_recall": 0.24263081943381815, "rougeL_recall_stderr": 0.004049339934122269, "rougeLsum_fmeasure": 0.08616972905837225, "rougeLsum_fmeasure_stderr": 0.0016929629302388789, "rougeLsum_precision": 0.05627985268944361, "rougeLsum_precision_stderr": 0.0012320876299857706, "rougeLsum_recall": 0.23899011890339064, "rougeLsum_recall_stderr": 0.003907803465727549}}, "1": {"PALM_prompt": {"bleu": 0.17248776305808664, "bleu_stderr": 0.016795777205216727, "rouge1_fmeasure": 0.07974711563423877, "rouge1_fmeasure_stderr": 0.001617468016734832, "rouge1_precision": 0.05236545537858825, "rouge1_precision_stderr": 0.001386009142665529, "rouge1_recall": 0.23952149357583752, "rouge1_recall_stderr": 0.003964748870758093, "rouge2_fmeasure": 0.03518570200418623, "rouge2_fmeasure_stderr": 0.001033184512581815, "rouge2_precision": 0.022770616206251718, "rouge2_precision_stderr": 0.0007222294749316485, "rouge2_recall": 0.10196623196520473, "rouge2_recall_stderr": 0.002694349434124461, "rougeL_fmeasure": 0.07801688157847861, "rougeL_fmeasure_stderr": 0.0015709474468090805, "rougeL_precision": 0.05119901012081213, "rougeL_precision_stderr": 0.0013530554709038835, "rougeL_recall": 0.23448910434022022, "rougeL_recall_stderr": 0.003865316164374324, "rougeLsum_fmeasure": 0.07704409187166199, "rougeLsum_fmeasure_stderr": 0.001581413920310377, "rougeLsum_precision": 0.050684197186769614, "rougeLsum_precision_stderr": 0.0013658718701688602, "rougeLsum_recall": 0.2299271822021421, "rougeLsum_recall_stderr": 0.003790698171203416}}, "2": {"PALM_prompt": {"bleu": 0.1740921740768521, "bleu_stderr": 0.02245852136332944, "rouge1_fmeasure": 0.0783917476843809, "rouge1_fmeasure_stderr": 0.0016004491851345573, "rouge1_precision": 0.05092812801458869, "rouge1_precision_stderr": 0.0011757535036755983, "rouge1_recall": 0.23029941909180796, "rouge1_recall_stderr": 0.003907134419847019, "rouge2_fmeasure": 0.03402987121771126, "rouge2_fmeasure_stderr": 0.0010090793783559, "rouge2_precision": 0.02209321030425811, "rouge2_precision_stderr": 0.0007051200067221665, "rouge2_recall": 0.09747850658202363, "rouge2_recall_stderr": 0.0025868322745989537, "rougeL_fmeasure": 0.07616316741043358, "rougeL_fmeasure_stderr": 0.0015474107907167969, "rougeL_precision": 0.049405325142194104, "rougeL_precision_stderr": 0.0011230235282996012, "rougeL_recall": 0.22350619198052418, "rougeL_recall_stderr": 0.0037728669875094984, "rougeLsum_fmeasure": 0.0753146662045205, "rougeLsum_fmeasure_stderr": 0.001548681630951826, "rougeLsum_precision": 0.048969182406106454, "rougeLsum_precision_stderr": 0.0011392876466907232, "rougeLsum_recall": 0.22012323480721052, "rougeLsum_recall_stderr": 0.0036888555418374417}}, "3": {"PALM_prompt": {"bleu": 0.1714945768037504, "bleu_stderr": 0.01842253558598964, "rouge1_fmeasure": 0.07920093236705308, "rouge1_fmeasure_stderr": 0.0015997822971242884, "rouge1_precision": 0.051722195443120254, "rouge1_precision_stderr": 0.0012608405872136094, "rouge1_recall": 0.23090502211754627, "rouge1_recall_stderr": 0.003988254256945192, "rouge2_fmeasure": 0.03455130562554136, "rouge2_fmeasure_stderr": 0.0010017255576368568, "rouge2_precision": 0.022332629611478836, "rouge2_precision_stderr": 0.0006936318961594426, "rouge2_recall": 0.09917949581608851, "rouge2_recall_stderr": 0.0026296436063060185, "rougeL_fmeasure": 0.07701796251510407, "rougeL_fmeasure_stderr": 0.0015576337881804197, "rougeL_precision": 0.050294237259899566, "rougeL_precision_stderr": 0.0012321105827659488, "rougeL_recall": 0.2246039480148206, "rougeL_recall_stderr": 0.003881559343048008, "rougeLsum_fmeasure": 0.07587583163215018, "rougeLsum_fmeasure_stderr": 0.0015390342361881646, "rougeLsum_precision": 0.049636917580444036, "rougeLsum_precision_stderr": 0.0012252524302076237, "rougeLsum_recall": 0.2198310121914411, "rougeLsum_recall_stderr": 0.0037347619995472055}}, "4": {"PALM_prompt": {"bleu": 0.2034962070733705, "bleu_stderr": 0.02512817739445294, "rouge1_fmeasure": 0.07919122114553223, "rouge1_fmeasure_stderr": 0.0015848708191778377, "rouge1_precision": 0.051358266508139044, "rouge1_precision_stderr": 0.0011609551790621932, "rouge1_recall": 0.2308574801783086, "rouge1_recall_stderr": 0.004058824692805349, "rouge2_fmeasure": 0.03439575337322024, "rouge2_fmeasure_stderr": 0.0009973225938294027, "rouge2_precision": 0.022201219787716792, "rouge2_precision_stderr": 0.0006921390509979684, "rouge2_recall": 0.1003506898317558, "rouge2_recall_stderr": 0.0026531796117823313, "rougeL_fmeasure": 0.07671934522499119, "rougeL_fmeasure_stderr": 0.0015289968779697359, "rougeL_precision": 0.0497139845045926, "rougeL_precision_stderr": 0.0011125764616461975, "rougeL_recall": 0.2236437131770874, "rougeL_recall_stderr": 0.003919177303008036, "rougeLsum_fmeasure": 0.07599501751355653, "rougeLsum_fmeasure_stderr": 0.0015187997824729323, "rougeLsum_precision": 0.04929778407737348, "rougeLsum_precision_stderr": 0.001109857222017746, "rougeLsum_recall": 0.2206395869777806, "rougeLsum_recall_stderr": 0.0037909884211304277}}, "5": {"PALM_prompt": {"bleu": 0.19761112575452508, "bleu_stderr": 0.019235465539383146, "rouge1_fmeasure": 0.08085878644777693, "rouge1_fmeasure_stderr": 0.0016289906236916856, "rouge1_precision": 0.052579153244265424, "rouge1_precision_stderr": 0.0011921637714878187, "rouge1_recall": 0.23535879092052564, "rouge1_recall_stderr": 0.0041071634335935175, "rouge2_fmeasure": 0.03502968692047543, "rouge2_fmeasure_stderr": 0.0010231866444401123, "rouge2_precision": 0.022671150422158365, "rouge2_precision_stderr": 0.000709183367358532, "rouge2_recall": 0.10160646453537654, "rouge2_recall_stderr": 0.002707170710882478, "rougeL_fmeasure": 0.07799816636434088, "rougeL_fmeasure_stderr": 0.0015628913834455332, "rougeL_precision": 0.050647877113144134, "rougeL_precision_stderr": 0.001139117442858445, "rougeL_recall": 0.2275596229219294, "rougeL_recall_stderr": 0.003973700969182821, "rougeLsum_fmeasure": 0.07750080873328469, "rougeLsum_fmeasure_stderr": 0.001570637280878507, "rougeLsum_precision": 0.050414491464882374, "rougeLsum_precision_stderr": 0.0011507171716506196, "rougeLsum_recall": 0.22494103941116889, "rougeLsum_recall_stderr": 0.0038872663249574415}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 0.45549657362697604, "bleu_stderr": 0.04494579402966035, "rouge1_fmeasure": 0.08341229365529593, "rouge1_fmeasure_stderr": 0.001426857348540837, "rouge1_precision": 0.07334286701079117, "rouge1_precision_stderr": 0.0014209616553442557, "rouge1_recall": 0.11781728898648271, "rouge1_recall_stderr": 0.002048222920943715, "rouge2_fmeasure": 0.00877170766892616, "rouge2_fmeasure_stderr": 0.0004188649332932362, "rouge2_precision": 0.007614750236728455, "rouge2_precision_stderr": 0.0003759989992852565, "rouge2_recall": 0.013390248256536815, "rouge2_recall_stderr": 0.000802825111743644, "rougeL_fmeasure": 0.07535689248472442, "rougeL_fmeasure_stderr": 0.0012035067304151934, "rougeL_precision": 0.06575808443610535, "rougeL_precision_stderr": 0.0011792063673595292, "rougeL_recall": 0.10763379445866503, "rougeL_recall_stderr": 0.0018034771592472351, "rougeLsum_fmeasure": 0.07735895099517735, "rougeLsum_fmeasure_stderr": 0.0013105886577076619, "rougeLsum_precision": 0.06788713608018786, "rougeLsum_precision_stderr": 0.0013039873220152538, "rougeLsum_recall": 0.10982226932159775, "rougeLsum_recall_stderr": 0.001904635097262412}}, "1": {"tldr_en": {"bleu": 0.4342251027303836, "bleu_stderr": 0.04156452799201429, "rouge1_fmeasure": 0.10367919976235754, "rouge1_fmeasure_stderr": 0.0013958183292936235, "rouge1_precision": 0.09377324802061514, "rouge1_precision_stderr": 0.0014183788800124697, "rouge1_recall": 0.14181618673418064, "rouge1_recall_stderr": 0.001999709640963563, "rouge2_fmeasure": 0.008218722219301039, "rouge2_fmeasure_stderr": 0.000410740325223334, "rouge2_precision": 0.007148933952844262, "rouge2_precision_stderr": 0.0003561943868556437, "rouge2_recall": 0.012216117762769886, "rouge2_recall_stderr": 0.0007004781288235548, "rougeL_fmeasure": 0.08313407829979999, "rougeL_fmeasure_stderr": 0.0009900147300689925, "rougeL_precision": 0.07479319586763088, "rougeL_precision_stderr": 0.0010178455763075111, "rougeL_recall": 0.1156679097936295, "rougeL_recall_stderr": 0.0015562821873263057, "rougeLsum_fmeasure": 0.09773392872756465, "rougeLsum_fmeasure_stderr": 0.001288712610683545, "rougeLsum_precision": 0.08828296890875521, "rougeLsum_precision_stderr": 0.0013091560120064637, "rougeLsum_recall": 0.134369194217566, "rougeLsum_recall_stderr": 0.0018930565890544265}}, "2": {"tldr_en": {"bleu": 0.4311595587295042, "bleu_stderr": 0.05311201104959919, "rouge1_fmeasure": 0.0960491519485715, "rouge1_fmeasure_stderr": 0.001321339124082858, "rouge1_precision": 0.09094561531157444, "rouge1_precision_stderr": 0.001456157079862017, "rouge1_recall": 0.12714249099405814, "rouge1_recall_stderr": 0.0018044241583523653, "rouge2_fmeasure": 0.007144914805869838, "rouge2_fmeasure_stderr": 0.00039460890013464246, "rouge2_precision": 0.006679127466354158, "rouge2_precision_stderr": 0.0004080467799025574, "rouge2_recall": 0.009968832268195739, "rouge2_recall_stderr": 0.0006285558582734068, "rougeL_fmeasure": 0.0815527888639139, "rougeL_fmeasure_stderr": 0.0010329480675385994, "rougeL_precision": 0.0766627034344775, "rougeL_precision_stderr": 0.0011486123351465115, "rougeL_recall": 0.10991861742729633, "rougeL_recall_stderr": 0.001524234817970641, "rougeLsum_fmeasure": 0.09016504132142787, "rougeLsum_fmeasure_stderr": 0.001216230722231443, "rougeLsum_precision": 0.08548630034360417, "rougeLsum_precision_stderr": 0.001361329490189725, "rougeLsum_recall": 0.11951122406072837, "rougeLsum_recall_stderr": 0.001667021801162019}}, "3": {"tldr_en": {"bleu": 0.4945744591124352, "bleu_stderr": 0.04112490496696115, "rouge1_fmeasure": 0.08383654969005261, "rouge1_fmeasure_stderr": 0.0014652925304525217, "rouge1_precision": 0.08508624990590842, "rouge1_precision_stderr": 0.0018605756778959563, "rouge1_recall": 0.11062172170633029, "rouge1_recall_stderr": 0.0020249317253040237, "rouge2_fmeasure": 0.008305764810262975, "rouge2_fmeasure_stderr": 0.00044353230048316586, "rouge2_precision": 0.008492614408520994, "rouge2_precision_stderr": 0.0005516284768616702, "rouge2_recall": 0.011602511796966591, "rouge2_recall_stderr": 0.0006906008777318128, "rougeL_fmeasure": 0.0733905772354038, "rougeL_fmeasure_stderr": 0.0012191555989918234, "rougeL_precision": 0.07393459992048995, "rougeL_precision_stderr": 0.0015722641471155711, "rougeL_recall": 0.09842577979303434, "rougeL_recall_stderr": 0.0017872350704345923, "rougeLsum_fmeasure": 0.07804802875444604, "rougeLsum_fmeasure_stderr": 0.0013485565230468167, "rougeLsum_precision": 0.07932181599326994, "rougeLsum_precision_stderr": 0.0017467205552871912, "rougeLsum_recall": 0.10333223732938228, "rougeLsum_recall_stderr": 0.0018867954598698327}}, "4": {"tldr_en": {"bleu": 0.1530683867528085, "bleu_stderr": 0.02304865388359797, "rouge1_fmeasure": 0.02787863882909451, "rouge1_fmeasure_stderr": 0.0011476158555931272, "rouge1_precision": 0.029004586891320194, "rouge1_precision_stderr": 0.001352875160534484, "rouge1_recall": 0.03766653675736129, "rouge1_recall_stderr": 0.0015747667565044058, "rouge2_fmeasure": 0.0030035258951381554, "rouge2_fmeasure_stderr": 0.00029734523782658787, "rouge2_precision": 0.0032795699585914644, "rouge2_precision_stderr": 0.00040022837086021713, "rouge2_recall": 0.004358266731620199, "rouge2_recall_stderr": 0.0004899708073278473, "rougeL_fmeasure": 0.025080472403581883, "rougeL_fmeasure_stderr": 0.0010020364992695604, "rougeL_precision": 0.025907175815949448, "rougeL_precision_stderr": 0.0011777866749689255, "rougeL_recall": 0.034451613541809104, "rougeL_recall_stderr": 0.001433290325245525, "rougeLsum_fmeasure": 0.025723161174858334, "rougeLsum_fmeasure_stderr": 0.001051539558458842, "rougeLsum_precision": 0.026962480466237466, "rougeLsum_precision_stderr": 0.0012715176574721234, "rougeLsum_recall": 0.03489216949042506, "rougeLsum_recall_stderr": 0.0014614184198085247}}, "5": {"tldr_en": {"bleu": 2.6420481653021324e-07, "bleu_stderr": 8.440587103346813e-07, "rouge1_fmeasure": 0.004344032704921915, "rouge1_fmeasure_stderr": 0.00048769273143641834, "rouge1_precision": 0.004746445530663895, "rouge1_precision_stderr": 0.0005682413591016014, "rouge1_recall": 0.005894862050119441, "rouge1_recall_stderr": 0.0006913140760157219, "rouge2_fmeasure": 0.00045459377689834523, "rouge2_fmeasure_stderr": 0.00010522677369073278, "rouge2_precision": 0.00044307036215439545, "rouge2_precision_stderr": 0.0001004207530010917, "rouge2_recall": 0.0006567436597417495, "rouge2_recall_stderr": 0.0001789950872715256, "rougeL_fmeasure": 0.00397221141689711, "rougeL_fmeasure_stderr": 0.00043801211935723714, "rougeL_precision": 0.004375980531133334, "rougeL_precision_stderr": 0.0005213355452750296, "rougeL_recall": 0.005438941343166284, "rougeL_recall_stderr": 0.000635980379876231, "rougeLsum_fmeasure": 0.004059385391361045, "rougeLsum_fmeasure_stderr": 0.0004536081698576505, "rougeLsum_precision": 0.004469825621096149, "rougeLsum_precision_stderr": 0.0005385326208085657, "rougeLsum_recall": 0.005512464579518613, "rougeLsum_recall_stderr": 0.0006502051982674122}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 0.5913211031265735, "bleu_stderr": 0.03714947332384052, "rouge1_fmeasure": 0.057779110162827135, "rouge1_fmeasure_stderr": 0.0018983542761005778, "rouge1_precision": 0.08058873419557824, "rouge1_precision_stderr": 0.0032571051434779103, "rouge1_recall": 0.06170899966771193, "rouge1_recall_stderr": 0.0020401649662685367, "rouge2_fmeasure": 0.01574069846670012, "rouge2_fmeasure_stderr": 0.0007666963173557723, "rouge2_precision": 0.02581229791743323, "rouge2_precision_stderr": 0.0015932358672241216, "rouge2_recall": 0.015789878220228096, "rouge2_recall_stderr": 0.0007208101441802875, "rougeL_fmeasure": 0.05161308138091674, "rougeL_fmeasure_stderr": 0.0016887665841841223, "rougeL_precision": 0.06923344159739564, "rougeL_precision_stderr": 0.0027564694592522388, "rougeL_recall": 0.05708601514024535, "rougeL_recall_stderr": 0.0019396856322431723, "rougeLsum_fmeasure": 0.04557718964996215, "rougeLsum_fmeasure_stderr": 0.0015307841093839293, "rougeLsum_precision": 0.06476318617469784, "rougeLsum_precision_stderr": 0.0027058888434037384, "rougeLsum_recall": 0.04796277405126694, "rougeLsum_recall_stderr": 0.0015865538487918089}}, "1": {"generate_text_restaurant": {"bleu": 3.7129526592901363, "bleu_stderr": 0.12569632286144403, "rouge1_fmeasure": 0.2847694417389595, "rouge1_fmeasure_stderr": 0.002040695495961792, "rouge1_precision": 0.33757710589590273, "rouge1_precision_stderr": 0.0027406590712350525, "rouge1_recall": 0.27297090245543504, "rouge1_recall_stderr": 0.0023001976057750945, "rouge2_fmeasure": 0.07230524386133462, "rouge2_fmeasure_stderr": 0.0013218286978928981, "rouge2_precision": 0.08861232666101841, "rouge2_precision_stderr": 0.001955054629766258, "rouge2_recall": 0.0692950974716365, "rouge2_recall_stderr": 0.0013251834733069956, "rougeL_fmeasure": 0.2070591630578462, "rougeL_fmeasure_stderr": 0.0015735715497195613, "rougeL_precision": 0.24807522899572165, "rougeL_precision_stderr": 0.0022872616728098504, "rougeL_recall": 0.19802854633734265, "rougeL_recall_stderr": 0.0017557148314317694, "rougeLsum_fmeasure": 0.23499367959584552, "rougeLsum_fmeasure_stderr": 0.0018330473958537736, "rougeLsum_precision": 0.28009204868111853, "rougeLsum_precision_stderr": 0.0025289533444105197, "rougeLsum_recall": 0.22495897081460098, "rougeLsum_recall_stderr": 0.002023117019898989}}, "2": {"generate_text_restaurant": {"bleu": 3.462491951458752, "bleu_stderr": 0.12932178148201628, "rouge1_fmeasure": 0.2636634022685503, "rouge1_fmeasure_stderr": 0.0024633437175046883, "rouge1_precision": 0.3164123369697131, "rouge1_precision_stderr": 0.0034775181138680792, "rouge1_recall": 0.2598745794315416, "rouge1_recall_stderr": 0.0027077821224868893, "rouge2_fmeasure": 0.06739797982186209, "rouge2_fmeasure_stderr": 0.001304257328973498, "rouge2_precision": 0.09371196451152501, "rouge2_precision_stderr": 0.0027593269396471646, "rouge2_recall": 0.06611332250980292, "rouge2_recall_stderr": 0.0013564334736745416, "rougeL_fmeasure": 0.19174609701033776, "rougeL_fmeasure_stderr": 0.0018513547565069343, "rougeL_precision": 0.23561396433786047, "rougeL_precision_stderr": 0.003025416778360062, "rougeL_recall": 0.1885327325268679, "rougeL_recall_stderr": 0.0020407871024373837, "rougeLsum_fmeasure": 0.21828405633341527, "rougeLsum_fmeasure_stderr": 0.002159399848115338, "rougeLsum_precision": 0.2651518207988083, "rougeLsum_precision_stderr": 0.003239070270009198, "rougeLsum_recall": 0.2148834164583239, "rougeLsum_recall_stderr": 0.0023603423697314426}}, "3": {"generate_text_restaurant": {"bleu": 3.1741335461954034, "bleu_stderr": 0.17453804662011688, "rouge1_fmeasure": 0.22501699083270982, "rouge1_fmeasure_stderr": 0.002791442670232082, "rouge1_precision": 0.29613934206804304, "rouge1_precision_stderr": 0.004451198145421545, "rouge1_recall": 0.22402208739224386, "rouge1_recall_stderr": 0.0030604385370891414, "rouge2_fmeasure": 0.06090634543978955, "rouge2_fmeasure_stderr": 0.0013142776626892116, "rouge2_precision": 0.11347371004313266, "rouge2_precision_stderr": 0.00401773931870467, "rouge2_recall": 0.06026738126786902, "rouge2_recall_stderr": 0.0014084605676934583, "rougeL_fmeasure": 0.16691558304091286, "rougeL_fmeasure_stderr": 0.0020999665649599486, "rougeL_precision": 0.231748269398982, "rougeL_precision_stderr": 0.004130558382055014, "rougeL_recall": 0.1649071700002769, "rougeL_recall_stderr": 0.0022872947567682198, "rougeLsum_fmeasure": 0.1886311008196838, "rougeLsum_fmeasure_stderr": 0.0024055139053464868, "rougeLsum_precision": 0.2556362993648324, "rougeLsum_precision_stderr": 0.00426628185731524, "rougeLsum_recall": 0.1872697901304377, "rougeLsum_recall_stderr": 0.002639460546137773}}, "4": {"generate_text_restaurant": {"bleu": 2.5346862761802016, "bleu_stderr": 0.12479053039126499, "rouge1_fmeasure": 0.21168945879136675, "rouge1_fmeasure_stderr": 0.0028856185357249834, "rouge1_precision": 0.31262165964896604, "rouge1_precision_stderr": 0.005047999697934806, "rouge1_recall": 0.1998465379109657, "rouge1_recall_stderr": 0.0030245662869133787, "rouge2_fmeasure": 0.06017636185529871, "rouge2_fmeasure_stderr": 0.0013291090944976105, "rouge2_precision": 0.1388925348990009, "rouge2_precision_stderr": 0.004790445901510994, "rouge2_recall": 0.05572347503457502, "rouge2_recall_stderr": 0.0013458562601325785, "rougeL_fmeasure": 0.16035037300782284, "rougeL_fmeasure_stderr": 0.0021851407273096933, "rougeL_precision": 0.253696212330601, "rougeL_precision_stderr": 0.004793613646337102, "rougeL_recall": 0.1499151276496759, "rougeL_recall_stderr": 0.002283351691483937, "rougeLsum_fmeasure": 0.1790062741536553, "rougeLsum_fmeasure_stderr": 0.002475640029677632, "rougeLsum_precision": 0.27512771480546755, "rougeLsum_precision_stderr": 0.004896008952662329, "rougeLsum_recall": 0.167957116845922, "rougeLsum_recall_stderr": 0.002583473884701478}}, "5": {"generate_text_restaurant": {"bleu": 2.4029523973418088, "bleu_stderr": 0.10662120310428724, "rouge1_fmeasure": 0.20707330180497546, "rouge1_fmeasure_stderr": 0.0029170286213066506, "rouge1_precision": 0.3260744222268218, "rouge1_precision_stderr": 0.005383991884322823, "rouge1_recall": 0.19321564713578962, "rouge1_recall_stderr": 0.0030459526877180125, "rouge2_fmeasure": 0.06161675662277744, "rouge2_fmeasure_stderr": 0.0013345748600508244, "rouge2_precision": 0.15881987477547233, "rouge2_precision_stderr": 0.005245943651586924, "rouge2_recall": 0.05630141761843497, "rouge2_recall_stderr": 0.0013558505278503214, "rougeL_fmeasure": 0.1584188601757907, "rougeL_fmeasure_stderr": 0.002197211198869285, "rougeL_precision": 0.2699334173910297, "rougeL_precision_stderr": 0.005178522168977427, "rougeL_recall": 0.14605882274029522, "rougeL_recall_stderr": 0.0022822110698795745, "rougeLsum_fmeasure": 0.17603000847259365, "rougeLsum_fmeasure_stderr": 0.0025014903430049438, "rougeLsum_precision": 0.29011330437239946, "rougeLsum_precision_stderr": 0.005265135503554526, "rougeLsum_recall": 0.16316644716956302, "rougeLsum_recall_stderr": 0.0026017608845607804}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 0.7376401152029481, "bleu_stderr": 0.0628176626129689, "rouge1_fmeasure": 0.13237124936382566, "rouge1_fmeasure_stderr": 0.002030318110427792, "rouge1_precision": 0.09522462576051435, "rouge1_precision_stderr": 0.0015560425280753526, "rouge1_recall": 0.23077643089582087, "rouge1_recall_stderr": 0.0034938489740306655, "rouge2_fmeasure": 0.020570971802076836, "rouge2_fmeasure_stderr": 0.0009528220708571039, "rouge2_precision": 0.014595206517583913, "rouge2_precision_stderr": 0.0006912221325529861, "rouge2_recall": 0.03697330908984525, "rouge2_recall_stderr": 0.0017262156912820615, "rougeL_fmeasure": 0.11697861341242026, "rougeL_fmeasure_stderr": 0.00168201405286699, "rougeL_precision": 0.08394414353545941, "rougeL_precision_stderr": 0.0012782778620016224, "rougeL_recall": 0.2050805363011997, "rougeL_recall_stderr": 0.002985550618482208, "rougeLsum_fmeasure": 0.1016392539142757, "rougeLsum_fmeasure_stderr": 0.0016105438516145294, "rougeLsum_precision": 0.0729362594061639, "rougeLsum_precision_stderr": 0.001222373131462693, "rougeLsum_recall": 0.17884876600176577, "rougeLsum_recall_stderr": 0.002908606459113646}}, "1": {"article_DOC_summary": {"bleu": 0.6702693938939304, "bleu_stderr": 0.07803616330676291, "rouge1_fmeasure": 0.1261090141950385, "rouge1_fmeasure_stderr": 0.002038712200394186, "rouge1_precision": 0.08993167019157738, "rouge1_precision_stderr": 0.0014994262109755236, "rouge1_recall": 0.2202950978484778, "rouge1_recall_stderr": 0.003503324448762303, "rouge2_fmeasure": 0.01664837239561062, "rouge2_fmeasure_stderr": 0.0008965007944526279, "rouge2_precision": 0.011719494218269198, "rouge2_precision_stderr": 0.0006320218059713919, "rouge2_recall": 0.02990361241325009, "rouge2_recall_stderr": 0.0016266190349500753, "rougeL_fmeasure": 0.11086188237596728, "rougeL_fmeasure_stderr": 0.0016892809835885039, "rougeL_precision": 0.07897756505863603, "rougeL_precision_stderr": 0.001243259199520003, "rougeL_recall": 0.19438615998458816, "rougeL_recall_stderr": 0.0029533459158402457, "rougeLsum_fmeasure": 0.0968810722814496, "rougeLsum_fmeasure_stderr": 0.0015962633597926558, "rougeLsum_precision": 0.06886845396607581, "rougeLsum_precision_stderr": 0.0011593284126768632, "rougeLsum_recall": 0.1706924968758996, "rougeLsum_recall_stderr": 0.0028400656934941242}}, "2": {"article_DOC_summary": {"bleu": 0.6510687810492491, "bleu_stderr": 0.09062967968019935, "rouge1_fmeasure": 0.12317572309436386, "rouge1_fmeasure_stderr": 0.0019948599460197793, "rouge1_precision": 0.08782405624093549, "rouge1_precision_stderr": 0.0014766059696371997, "rouge1_recall": 0.2160456627932237, "rouge1_recall_stderr": 0.003441332688585382, "rouge2_fmeasure": 0.017543316635811195, "rouge2_fmeasure_stderr": 0.0009239971756884746, "rouge2_precision": 0.012333383775107388, "rouge2_precision_stderr": 0.0006521057452803417, "rouge2_recall": 0.03179899644401759, "rouge2_recall_stderr": 0.001722233233752029, "rougeL_fmeasure": 0.1092412067078929, "rougeL_fmeasure_stderr": 0.0016683921666508167, "rougeL_precision": 0.07776394031041892, "rougeL_precision_stderr": 0.0012329815052524796, "rougeL_recall": 0.19243820960438146, "rougeL_recall_stderr": 0.0029224083235941836, "rougeLsum_fmeasure": 0.09510788828477747, "rougeLsum_fmeasure_stderr": 0.0015320545647342031, "rougeLsum_precision": 0.0676638257670816, "rougeLsum_precision_stderr": 0.0011247421201455816, "rougeLsum_recall": 0.16808542682817457, "rougeLsum_recall_stderr": 0.0027515286144884723}}, "3": {"article_DOC_summary": {"bleu": 0.694494856973092, "bleu_stderr": 0.09315806446376278, "rouge1_fmeasure": 0.12144582095736131, "rouge1_fmeasure_stderr": 0.002114335303171797, "rouge1_precision": 0.08897486239114341, "rouge1_precision_stderr": 0.0017009074998627915, "rouge1_recall": 0.2091191397530532, "rouge1_recall_stderr": 0.003614643324036501, "rouge2_fmeasure": 0.01754230953552776, "rouge2_fmeasure_stderr": 0.0009355858622010496, "rouge2_precision": 0.01259505859479755, "rouge2_precision_stderr": 0.00069046073565253, "rouge2_recall": 0.03122412533566738, "rouge2_recall_stderr": 0.001678680477457119, "rougeL_fmeasure": 0.10734285670078089, "rougeL_fmeasure_stderr": 0.001745478308264722, "rougeL_precision": 0.07838071014182374, "rougeL_precision_stderr": 0.001396156009643834, "rougeL_recall": 0.18573352825783288, "rougeL_recall_stderr": 0.003039958738646297, "rougeLsum_fmeasure": 0.0931643849311634, "rougeLsum_fmeasure_stderr": 0.0016188390418371856, "rougeLsum_precision": 0.06823456301932596, "rougeLsum_precision_stderr": 0.0013342812919056666, "rougeLsum_recall": 0.16149631288933364, "rougeLsum_recall_stderr": 0.0028294650899024533}}, "4": {"article_DOC_summary": {"bleu": 0.4634958897519297, "bleu_stderr": 0.1078601254660865, "rouge1_fmeasure": 0.03483851864300909, "rouge1_fmeasure_stderr": 0.0020507945477994454, "rouge1_precision": 0.03203907131106538, "rouge1_precision_stderr": 0.0023266110566352526, "rouge1_recall": 0.05290796868421392, "rouge1_recall_stderr": 0.0030930646890496914, "rouge2_fmeasure": 0.005528078242750013, "rouge2_fmeasure_stderr": 0.0007181419240142681, "rouge2_precision": 0.005078726019956826, "rouge2_precision_stderr": 0.0008900002887502463, "rouge2_recall": 0.008305472613043527, "rouge2_recall_stderr": 0.0009638105580648599, "rougeL_fmeasure": 0.03034968255806866, "rougeL_fmeasure_stderr": 0.0017240783735724304, "rougeL_precision": 0.02793525308488722, "rougeL_precision_stderr": 0.0020306661448374493, "rougeL_recall": 0.04674043181938062, "rougeL_recall_stderr": 0.002691339349960188, "rougeLsum_fmeasure": 0.027467434064520325, "rougeLsum_fmeasure_stderr": 0.0016200826802563308, "rougeLsum_precision": 0.02589849825305961, "rougeLsum_precision_stderr": 0.0020040582236913592, "rougeLsum_recall": 0.04190637777442485, "rougeLsum_recall_stderr": 0.0024786007167556857}}, "5": {"article_DOC_summary": {"bleu": 9.409589204950959e-38, "bleu_stderr": 2.250977748694596e-32, "rouge1_fmeasure": 0.0024866753710265106, "rouge1_fmeasure_stderr": 0.0006684639506190258, "rouge1_precision": 0.002739129716464841, "rouge1_precision_stderr": 0.0007581348910112568, "rouge1_recall": 0.0023745897883191555, "rouge1_recall_stderr": 0.0006352252378761389, "rouge2_fmeasure": 0.0004287346762241937, "rouge2_fmeasure_stderr": 0.0002053885706784475, "rouge2_precision": 0.00047501133100467176, "rouge2_precision_stderr": 0.00021640271325881454, "rouge2_recall": 0.0003994503753543417, "rouge2_recall_stderr": 0.0002011697037898624, "rougeL_fmeasure": 0.0019488148577111895, "rougeL_fmeasure_stderr": 0.000526299388991192, "rougeL_precision": 0.002126911972697357, "rougeL_precision_stderr": 0.0005869633815800918, "rougeL_recall": 0.0018836680062018954, "rougeL_recall_stderr": 0.0005134831663617108, "rougeLsum_fmeasure": 0.0020723992306803042, "rougeLsum_fmeasure_stderr": 0.0005716554317274563, "rougeLsum_precision": 0.0022796750730625366, "rougeLsum_precision_stderr": 0.0006491389537922883, "rougeLsum_recall": 0.001987887384982107, "rougeLsum_recall_stderr": 0.0005467706089350191}}}} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8bb165f0daac7ba28a6ba2bdbb1119d1baedac1f --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.24826051946595584, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.022223944257084786 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.058163199053238857, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0012927818826889883 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2464607565329814, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004099130993659631 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08900765115929998, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017737031911390954 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.027958228185354133, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008197899214780566 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1212206967206692, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002879981343925423 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04299602830782456, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011529371606547893 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05687669768133077, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012334222658519987 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.24263081943381815, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004049339934122269 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.08719970330894132, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017067065975610177 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05627985268944361, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012320876299857706 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.23899011890339064, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.003907803465727549 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.08616972905837225, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016929629302388789 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b645cc058285fd72e691c5396a7b7f247ae52903 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.17248776305808664, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.016795777205216727 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05236545537858825, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001386009142665529 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.23952149357583752, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003964748870758093 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.07974711563423877, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001617468016734832 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.022770616206251718, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007222294749316485 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.10196623196520473, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002694349434124461 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03518570200418623, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001033184512581815 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05119901012081213, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013530554709038835 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.23448910434022022, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003865316164374324 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07801688157847861, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015709474468090805 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.050684197186769614, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013658718701688602 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2299271822021421, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.003790698171203416 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07704409187166199, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001581413920310377 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..40974c197ccd1b493af52d7464cbede8e2acfd73 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.1740921740768521, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02245852136332944 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05092812801458869, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011757535036755983 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.23029941909180796, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003907134419847019 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.0783917476843809, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016004491851345573 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02209321030425811, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007051200067221665 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.09747850658202363, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0025868322745989537 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03402987121771126, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010090793783559 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.049405325142194104, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011230235282996012 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.22350619198052418, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0037728669875094984 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07616316741043358, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015474107907167969 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.048969182406106454, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011392876466907232 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.22012323480721052, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0036888555418374417 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.0753146662045205, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001548681630951826 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..5cd421c42488235569b6b980094ea88c158e7946 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.1714945768037504, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.01842253558598964 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.051722195443120254, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0012608405872136094 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.23090502211754627, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003988254256945192 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.07920093236705308, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015997822971242884 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.022332629611478836, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006936318961594426 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.09917949581608851, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0026296436063060185 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03455130562554136, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010017255576368568 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.050294237259899566, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012321105827659488 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2246039480148206, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003881559343048008 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07701796251510407, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015576337881804197 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.049636917580444036, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012252524302076237 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2198310121914411, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0037347619995472055 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07587583163215018, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015390342361881646 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..23c6a986b818ef7f17b92d06159d33860d8d7210 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.2034962070733705, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02512817739445294 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.051358266508139044, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011609551790621932 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2308574801783086, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004058824692805349 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.07919122114553223, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015848708191778377 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.022201219787716792, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006921390509979684 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1003506898317558, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0026531796117823313 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03439575337322024, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009973225938294027 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0497139845045926, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011125764616461975 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2236437131770874, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003919177303008036 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07671934522499119, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015289968779697359 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.04929778407737348, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001109857222017746 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2206395869777806, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0037909884211304277 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07599501751355653, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015187997824729323 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..1fcae413ff70d379420e57896c28018d0a505964 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.19761112575452508, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.019235465539383146 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.052579153244265424, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011921637714878187 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.23535879092052564, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0041071634335935175 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08085878644777693, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016289906236916856 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.022671150422158365, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000709183367358532 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.10160646453537654, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002707170710882478 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03502968692047543, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010231866444401123 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.050647877113144134, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001139117442858445 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2275596229219294, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003973700969182821 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07799816636434088, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015628913834455332 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.050414491464882374, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011507171716506196 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.22494103941116889, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0038872663249574415 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07750080873328469, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001570637280878507 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..da636004f876158f035732987f171311a320212f --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.07334286701079117, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014209616553442557 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.11781728898648271, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002048222920943715 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.08341229365529593, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001426857348540837 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.007614750236728455, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0003759989992852565 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.013390248256536815, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.000802825111743644 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.00877170766892616, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004188649332932362 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.06575808443610535, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011792063673595292 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.10763379445866503, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018034771592472351 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.07535689248472442, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012035067304151934 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.06788713608018786, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013039873220152538 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.10982226932159775, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.001904635097262412 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.07735895099517735, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013105886577076619 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.45549657362697604, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04494579402966035 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e6cebcfc8ef927d72f27466fd56cc4e0d6c67bf8 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.09377324802061514, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014183788800124697 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.14181618673418064, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.001999709640963563 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.10367919976235754, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0013958183292936235 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.007148933952844262, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0003561943868556437 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.012216117762769886, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0007004781288235548 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.008218722219301039, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.000410740325223334 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07479319586763088, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010178455763075111 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.1156679097936295, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0015562821873263057 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.08313407829979999, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0009900147300689925 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.08828296890875521, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013091560120064637 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.134369194217566, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0018930565890544265 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09773392872756465, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001288712610683545 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4342251027303836, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04156452799201429 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f5e6a9de346d9545446a9b670627fe6c6ffb16 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.09094561531157444, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001456157079862017 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.12714249099405814, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0018044241583523653 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.0960491519485715, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001321339124082858 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.006679127466354158, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0004080467799025574 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.009968832268195739, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0006285558582734068 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.007144914805869838, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00039460890013464246 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.0766627034344775, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011486123351465115 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.10991861742729633, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.001524234817970641 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0815527888639139, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010329480675385994 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.08548630034360417, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001361329490189725 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.11951122406072837, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.001667021801162019 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09016504132142787, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001216230722231443 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4311595587295042, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05311201104959919 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f643e13d911108c68e8cf8c77e040aa87eb09099 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08508624990590842, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018605756778959563 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.11062172170633029, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0020249317253040237 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.08383654969005261, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014652925304525217 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.008492614408520994, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005516284768616702 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.011602511796966591, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0006906008777318128 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.008305764810262975, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00044353230048316586 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07393459992048995, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0015722641471155711 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.09842577979303434, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0017872350704345923 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0733905772354038, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012191555989918234 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.07932181599326994, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0017467205552871912 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.10333223732938228, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0018867954598698327 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.07804802875444604, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013485565230468167 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4945744591124352, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04112490496696115 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e4aa35413c416ce15e022eda255f1dfd616c47d6 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.029004586891320194, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001352875160534484 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.03766653675736129, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0015747667565044058 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.02787863882909451, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0011476158555931272 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0032795699585914644, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00040022837086021713 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.004358266731620199, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0004899708073278473 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0030035258951381554, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00029734523782658787 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.025907175815949448, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011777866749689255 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.034451613541809104, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.001433290325245525 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.025080472403581883, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010020364992695604 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.026962480466237466, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012715176574721234 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.03489216949042506, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0014614184198085247 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.025723161174858334, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001051539558458842 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.1530683867528085, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02304865388359797 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c39b0b04940172602a00cb8248cba4411ef20274 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.004746445530663895, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0005682413591016014 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.005894862050119441, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0006913140760157219 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.004344032704921915, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.00048769273143641834 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.00044307036215439545, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0001004207530010917 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0006567436597417495, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0001789950872715256 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.00045459377689834523, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00010522677369073278 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.004375980531133334, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0005213355452750296 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.005438941343166284, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.000635980379876231 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.00397221141689711, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.00043801211935723714 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.004469825621096149, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0005385326208085657 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.005512464579518613, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0006502051982674122 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.004059385391361045, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0004536081698576505 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 2.6420481653021324e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 8.440587103346813e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1d346de01140ecd6e7507a9cfe78c5b28ef2072a --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 0.5913211031265735, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.03714947332384052 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.08058873419557824, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0032571051434779103 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.06170899966771193, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0020401649662685367 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.057779110162827135, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0018983542761005778 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.02581229791743323, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0015932358672241216 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.015789878220228096, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0007208101441802875 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.01574069846670012, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0007666963173557723 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.06923344159739564, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0027564694592522388 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.05708601514024535, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0019396856322431723 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.05161308138091674, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0016887665841841223 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.06476318617469784, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0027058888434037384 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.04796277405126694, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0015865538487918089 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.04557718964996215, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015307841093839293 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a271fc7653486d9f85c6662cbfa4709b515ef312 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.7129526592901363, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12569632286144403 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.33757710589590273, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0027406590712350525 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.27297090245543504, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0023001976057750945 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.2847694417389595, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002040695495961792 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.08861232666101841, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.001955054629766258 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.0692950974716365, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013251834733069956 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.07230524386133462, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013218286978928981 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.24807522899572165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0022872616728098504 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.19802854633734265, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0017557148314317694 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.2070591630578462, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0015735715497195613 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.28009204868111853, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0025289533444105197 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.22495897081460098, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002023117019898989 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.23499367959584552, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018330473958537736 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..5fb8c9cabbfdcb11b61e40276991843d4bd1dd95 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.462491951458752, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12932178148201628 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.3164123369697131, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0034775181138680792 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.2598745794315416, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027077821224868893 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.2636634022685503, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0024633437175046883 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.09371196451152501, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0027593269396471646 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.06611332250980292, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013564334736745416 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06739797982186209, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001304257328973498 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.23561396433786047, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.003025416778360062 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.1885327325268679, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0020407871024373837 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.19174609701033776, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018513547565069343 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2651518207988083, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.003239070270009198 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.2148834164583239, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0023603423697314426 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.21828405633341527, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002159399848115338 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7f55248fa5bc49f7887d5111c3a47969a4bcf8f6 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.1741335461954034, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.17453804662011688 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.29613934206804304, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.004451198145421545 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.22402208739224386, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0030604385370891414 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.22501699083270982, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002791442670232082 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.11347371004313266, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.00401773931870467 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.06026738126786902, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0014084605676934583 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06090634543978955, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013142776626892116 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.231748269398982, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.004130558382055014 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.1649071700002769, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022872947567682198 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.16691558304091286, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020999665649599486 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2556362993648324, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.00426628185731524 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.1872697901304377, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002639460546137773 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.1886311008196838, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0024055139053464868 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..7d8480e8435cd1e8712d5b54fc1b46431ffc8710 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.5346862761802016, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12479053039126499 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.31262165964896604, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.005047999697934806 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1998465379109657, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0030245662869133787 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.21168945879136675, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0028856185357249834 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.1388925348990009, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.004790445901510994 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.05572347503457502, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013458562601325785 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06017636185529871, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013291090944976105 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.253696212330601, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.004793613646337102 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.1499151276496759, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002283351691483937 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.16035037300782284, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0021851407273096933 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.27512771480546755, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.004896008952662329 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.167957116845922, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002583473884701478 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.1790062741536553, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002475640029677632 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..59c1b0e4b9ca033ce0e4fd2d456053e22e08054d --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.4029523973418088, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.10662120310428724 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.3260744222268218, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.005383991884322823 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.19321564713578962, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0030459526877180125 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.20707330180497546, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0029170286213066506 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.15881987477547233, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.005245943651586924 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.05630141761843497, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013558505278503214 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06161675662277744, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013345748600508244 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.2699334173910297, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.005178522168977427 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.14605882274029522, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022822110698795745 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.1584188601757907, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.002197211198869285 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.29011330437239946, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.005265135503554526 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.16316644716956302, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0026017608845607804 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.17603000847259365, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0025014903430049438 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_0.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..525578ab788c3ce9e36e376b216114794531f785 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09522462576051435, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0015560425280753526 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.23077643089582087, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0034938489740306655 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.13237124936382566, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002030318110427792 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.014595206517583913, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006912221325529861 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03697330908984525, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0017262156912820615 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.020570971802076836, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009528220708571039 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.08394414353545941, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012782778620016224 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2050805363011997, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002985550618482208 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11697861341242026, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.00168201405286699 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0729362594061639, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001222373131462693 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.17884876600176577, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002908606459113646 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1016392539142757, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016105438516145294 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.7376401152029481, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.0628176626129689 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_1.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..71c9c4d2c79b933f23a64e9f8358100fd01e26f3 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08993167019157738, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0014994262109755236 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2202950978484778, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003503324448762303 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.1261090141950385, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002038712200394186 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011719494218269198, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006320218059713919 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.02990361241325009, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0016266190349500753 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.01664837239561062, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0008965007944526279 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07897756505863603, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001243259199520003 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.19438615998458816, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029533459158402457 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11086188237596728, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016892809835885039 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06886845396607581, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011593284126768632 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1706924968758996, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0028400656934941242 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0968810722814496, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015962633597926558 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6702693938939304, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.07803616330676291 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_2.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..077bddfed19b1c84e17503481e3328b2fb9fc412 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08782405624093549, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0014766059696371997 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2160456627932237, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003441332688585382 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.12317572309436386, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0019948599460197793 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.012333383775107388, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006521057452803417 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03179899644401759, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.001722233233752029 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.017543316635811195, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009239971756884746 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07776394031041892, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012329815052524796 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.19243820960438146, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029224083235941836 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.1092412067078929, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016683921666508167 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0676638257670816, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011247421201455816 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.16808542682817457, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0027515286144884723 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.09510788828477747, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015320545647342031 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6510687810492491, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.09062967968019935 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_3.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..64a4b75dac46d09d78c5bc74ae32c9d1e4185287 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08897486239114341, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0017009074998627915 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2091191397530532, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003614643324036501 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.12144582095736131, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002114335303171797 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.01259505859479755, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.00069046073565253 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03122412533566738, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.001678680477457119 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.01754230953552776, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009355858622010496 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07838071014182374, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001396156009643834 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.18573352825783288, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003039958738646297 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10734285670078089, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001745478308264722 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06823456301932596, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0013342812919056666 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.16149631288933364, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0028294650899024533 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0931643849311634, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016188390418371856 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.694494856973092, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.09315806446376278 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_4.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c32d527213fbbeaa47f992f6a158a4fc83dce752 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.03203907131106538, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0023266110566352526 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.05290796868421392, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0030930646890496914 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.03483851864300909, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0020507945477994454 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.005078726019956826, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0008900002887502463 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.008305472613043527, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0009638105580648599 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.005528078242750013, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0007181419240142681 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.02793525308488722, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0020306661448374493 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04674043181938062, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002691339349960188 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.03034968255806866, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0017240783735724304 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.02589849825305961, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0020040582236913592 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.04190637777442485, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0024786007167556857 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.027467434064520325, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016200826802563308 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.4634958897519297, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.1078601254660865 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_5.json b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c40d2b811331835fdf271911299baeb57a087664 --- /dev/null +++ b/619m2b71b5/evaluation/generation/slim.619m2b71b5_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.002739129716464841, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0007581348910112568 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0023745897883191555, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0006352252378761389 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0024866753710265106, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0006684639506190258 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00047501133100467176, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.00021640271325881454 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0003994503753543417, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0002011697037898624 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0004287346762241937, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0002053885706784475 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.002126911972697357, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0005869633815800918 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0018836680062018954, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0005134831663617108 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0019488148577111895, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.000526299388991192 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0022796750730625366, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0006491389537922883 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.001987887384982107, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0005467706089350191 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0020723992306803042, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0005716554317274563 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 9.409589204950959e-38, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 2.250977748694596e-32 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b71b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_0.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..54034ca0ec467f39d283f64deea05dbe7aa1c2e8 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.335,0.014933117490932575,0 +anli_r2,acc,0.343,0.015019206922356953,0 +anli_r3,acc,0.32916666666666666,0.013570806258433621,0 +arc_challenge,acc,0.1680887372013652,0.01092771504612486,0 +arc_challenge,acc_norm,0.22610921501706485,0.01222420209706328,0 +arc_easy,acc,0.38636363636363635,0.009991296778159629,0 +arc_easy,acc_norm,0.35563973063973064,0.009822854395535487,0 +boolq,acc,0.5926605504587156,0.008593573302607049,1 +cb,acc,0.4642857142857143,0.0672477765493766,1 +cb,f1,0.2985842985842986,,1 +copa,acc,0.61,0.04902071300001974,0 +hellaswag,acc,0.28072097191794465,0.004484330827465548,0 +hellaswag,acc_norm,0.28759211312487554,0.0045171484341805105,0 +piqa,acc,0.6311207834602829,0.011257546676908805,0 +piqa,acc_norm,0.6153427638737758,0.011351177743205583,0 +rte,acc,0.5342960288808665,0.030025579819366426,0 +sciq,acc,0.659,0.014998131348402702,0 +sciq,acc_norm,0.573,0.015649789644462224,0 +storycloze_2016,acc,0.5708177445216461,0.011445869980247972,0 +winogrande,acc,0.5224940805051302,0.01403825782405988,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_0.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_0.json new file mode 100644 index 0000000000000000000000000000000000000000..85a311b9b103e1876c6dea45f2ca158464dd6221 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932575 + }, + "anli_r2": { + "acc": 0.343, + "acc_stderr": 0.015019206922356953 + }, + "anli_r3": { + "acc": 0.32916666666666666, + "acc_stderr": 0.013570806258433621 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.0672477765493766, + "f1": 0.2985842985842986 + }, + "copa": { + "acc": 0.61, + "acc_stderr": 0.04902071300001974 + }, + "hellaswag": { + "acc": 0.28072097191794465, + "acc_stderr": 0.004484330827465548, + "acc_norm": 0.28759211312487554, + "acc_norm_stderr": 0.0045171484341805105 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.5224940805051302, + "acc_stderr": 0.01403825782405988 + }, + "storycloze_2016": { + "acc": 0.5708177445216461, + "acc_stderr": 0.011445869980247972 + }, + "boolq": { + "acc": 0.5926605504587156, + "acc_stderr": 0.008593573302607049 + }, + "arc_easy": { + "acc": 0.38636363636363635, + "acc_stderr": 0.009991296778159629, + "acc_norm": 0.35563973063973064, + "acc_norm_stderr": 0.009822854395535487 + }, + "arc_challenge": { + "acc": 0.1680887372013652, + "acc_stderr": 0.01092771504612486, + "acc_norm": 0.22610921501706485, + "acc_norm_stderr": 0.01222420209706328 + }, + "sciq": { + "acc": 0.659, + "acc_stderr": 0.014998131348402702, + "acc_norm": 0.573, + "acc_norm_stderr": 0.015649789644462224 + }, + "piqa": { + "acc": 0.6311207834602829, + "acc_stderr": 0.011257546676908805, + "acc_norm": 0.6153427638737758, + "acc_norm_stderr": 0.011351177743205583 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_1.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c5fab259a61bddb5e5b43d23bafa6cb4c8c19be --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.336,0.014944140233795023,0 +anli_r2,acc,0.33,0.01487687202745674,0 +anli_r3,acc,0.3333333333333333,0.0136139500102256,0 +arc_challenge,acc,0.17491467576791808,0.01110156250182823,0 +arc_challenge,acc_norm,0.2226962457337884,0.012158314774829933,0 +arc_easy,acc,0.3838383838383838,0.009979061846649305,0 +arc_easy,acc_norm,0.3484848484848485,0.009777377947106538,0 +boolq,acc,0.5321100917431193,0.008727003026917802,1 +cb,acc,0.5,0.06741998624632421,1 +cb,f1,0.34717335209946043,,1 +copa,acc,0.59,0.04943110704237101,0 +hellaswag,acc,0.27942640908185623,0.004478003326528241,0 +hellaswag,acc_norm,0.28978291177056364,0.0045273436511308095,0 +piqa,acc,0.6235038084874864,0.011304341550126735,0 +piqa,acc_norm,0.6175190424374319,0.011339019654272347,0 +rte,acc,0.5126353790613718,0.030086851767188564,0 +sciq,acc,0.65,0.015090650341444233,0 +sciq,acc_norm,0.581,0.015610338967577792,0 +storycloze_2016,acc,0.5654730090860502,0.011462873926422448,0 +winogrande,acc,0.5169692186266772,0.014044390401612981,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_1.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_1.json new file mode 100644 index 0000000000000000000000000000000000000000..be2585bca878a2e3a15e118ce679f788c0c11497 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.336, + "acc_stderr": 0.014944140233795023 + }, + "anli_r2": { + "acc": 0.33, + "acc_stderr": 0.01487687202745674 + }, + "anli_r3": { + "acc": 0.3333333333333333, + "acc_stderr": 0.0136139500102256 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.34717335209946043 + }, + "copa": { + "acc": 0.59, + "acc_stderr": 0.04943110704237101 + }, + "hellaswag": { + "acc": 0.27942640908185623, + "acc_stderr": 0.004478003326528241, + "acc_norm": 0.28978291177056364, + "acc_norm_stderr": 0.0045273436511308095 + }, + "rte": { + "acc": 0.5126353790613718, + "acc_stderr": 0.030086851767188564 + }, + "winogrande": { + "acc": 0.5169692186266772, + "acc_stderr": 0.014044390401612981 + }, + "storycloze_2016": { + "acc": 0.5654730090860502, + "acc_stderr": 0.011462873926422448 + }, + "boolq": { + "acc": 0.5321100917431193, + "acc_stderr": 0.008727003026917802 + }, + "arc_easy": { + "acc": 0.3838383838383838, + "acc_stderr": 0.009979061846649305, + "acc_norm": 0.3484848484848485, + "acc_norm_stderr": 0.009777377947106538 + }, + "arc_challenge": { + "acc": 0.17491467576791808, + "acc_stderr": 0.01110156250182823, + "acc_norm": 0.2226962457337884, + "acc_norm_stderr": 0.012158314774829933 + }, + "sciq": { + "acc": 0.65, + "acc_stderr": 0.015090650341444233, + "acc_norm": 0.581, + "acc_norm_stderr": 0.015610338967577792 + }, + "piqa": { + "acc": 0.6235038084874864, + "acc_stderr": 0.011304341550126735, + "acc_norm": 0.6175190424374319, + "acc_norm_stderr": 0.011339019654272347 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_2.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..cadd2f7c76805bee881387d9c73cffd53638b3fd --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.341,0.01499813134840269,0 +anli_r2,acc,0.34,0.014987482264363937,0 +anli_r3,acc,0.345,0.013728421539454878,0 +arc_challenge,acc,0.1766211604095563,0.011144042769316503,0 +arc_challenge,acc_norm,0.21331058020477817,0.011970971742326334,0 +arc_easy,acc,0.3779461279461279,0.009949405744045469,0 +arc_easy,acc_norm,0.343013468013468,0.009740965666489224,0 +boolq,acc,0.5593272171253822,0.00868327649582901,1 +cb,acc,0.44642857142857145,0.06703189227942398,1 +cb,f1,0.3104093567251462,,1 +copa,acc,0.63,0.04852365870939099,0 +hellaswag,acc,0.27703644692292373,0.004466200055292544,0 +hellaswag,acc_norm,0.2898824935271858,0.00452780401625378,0 +piqa,acc,0.6229597388465724,0.011307569752543902,0 +piqa,acc_norm,0.6115342763873776,0.011371877593210252,0 +rte,acc,0.48375451263537905,0.030080573208738064,0 +sciq,acc,0.65,0.015090650341444233,0 +sciq,acc_norm,0.613,0.015410011955493933,0 +storycloze_2016,acc,0.5606627471940139,0.01147701798230878,0 +winogrande,acc,0.5019731649565904,0.014052376259225629,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_2.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c007008ab8b8f9b49db59877035f63ce9598ac3e --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.341, + "acc_stderr": 0.01499813134840269 + }, + "anli_r2": { + "acc": 0.34, + "acc_stderr": 0.014987482264363937 + }, + "anli_r3": { + "acc": 0.345, + "acc_stderr": 0.013728421539454878 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.3104093567251462 + }, + "copa": { + "acc": 0.63, + "acc_stderr": 0.04852365870939099 + }, + "hellaswag": { + "acc": 0.27703644692292373, + "acc_stderr": 0.004466200055292544, + "acc_norm": 0.2898824935271858, + "acc_norm_stderr": 0.00452780401625378 + }, + "rte": { + "acc": 0.48375451263537905, + "acc_stderr": 0.030080573208738064 + }, + "winogrande": { + "acc": 0.5019731649565904, + "acc_stderr": 0.014052376259225629 + }, + "storycloze_2016": { + "acc": 0.5606627471940139, + "acc_stderr": 0.01147701798230878 + }, + "boolq": { + "acc": 0.5593272171253822, + "acc_stderr": 0.00868327649582901 + }, + "arc_easy": { + "acc": 0.3779461279461279, + "acc_stderr": 0.009949405744045469, + "acc_norm": 0.343013468013468, + "acc_norm_stderr": 0.009740965666489224 + }, + "arc_challenge": { + "acc": 0.1766211604095563, + "acc_stderr": 0.011144042769316503, + "acc_norm": 0.21331058020477817, + "acc_norm_stderr": 0.011970971742326334 + }, + "sciq": { + "acc": 0.65, + "acc_stderr": 0.015090650341444233, + "acc_norm": 0.613, + "acc_norm_stderr": 0.015410011955493933 + }, + "piqa": { + "acc": 0.6229597388465724, + "acc_stderr": 0.011307569752543902, + "acc_norm": 0.6115342763873776, + "acc_norm_stderr": 0.011371877593210252 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_3.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..206f62c5e6f7f3c5708f346edc4e038cf98ee71b --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.319,0.014746404865473465,0 +anli_r2,acc,0.351,0.015100563798316402,0 +anli_r3,acc,0.3308333333333333,0.013588208070708993,0 +arc_challenge,acc,0.17406143344709898,0.01108017712948222,0 +arc_challenge,acc_norm,0.2226962457337884,0.012158314774829931,0 +arc_easy,acc,0.35774410774410775,0.00983577275734336,0 +arc_easy,acc_norm,0.3409090909090909,0.009726579593424019,0 +boolq,acc,0.5740061162079511,0.008648732832949143,1 +cb,acc,0.375,0.06527912098338669,1 +cb,f1,0.2603800140745954,,1 +copa,acc,0.59,0.049431107042371025,0 +hellaswag,acc,0.2800238996215893,0.00448092945028156,0 +hellaswag,acc_norm,0.28809002190798644,0.004519476835646766,0 +piqa,acc,0.6240478781284005,0.011301098166895732,0 +piqa,acc_norm,0.6169749727965179,0.011342081709082847,0 +rte,acc,0.5306859205776173,0.030039730592197812,0 +sciq,acc,0.646,0.015129868238451773,0 +sciq,acc_norm,0.597,0.01551875741906653,0 +storycloze_2016,acc,0.5606627471940139,0.01147701798230878,0 +winogrande,acc,0.5019731649565904,0.014052376259225629,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_3.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_3.json new file mode 100644 index 0000000000000000000000000000000000000000..108769d3d637e3cc24f9d1e0172664557c127a0a --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.319, + "acc_stderr": 0.014746404865473465 + }, + "anli_r2": { + "acc": 0.351, + "acc_stderr": 0.015100563798316402 + }, + "anli_r3": { + "acc": 0.3308333333333333, + "acc_stderr": 0.013588208070708993 + }, + "cb": { + "acc": 0.375, + "acc_stderr": 0.06527912098338669, + "f1": 0.2603800140745954 + }, + "copa": { + "acc": 0.59, + "acc_stderr": 0.049431107042371025 + }, + "hellaswag": { + "acc": 0.2800238996215893, + "acc_stderr": 0.00448092945028156, + "acc_norm": 0.28809002190798644, + "acc_norm_stderr": 0.004519476835646766 + }, + "rte": { + "acc": 0.5306859205776173, + "acc_stderr": 0.030039730592197812 + }, + "winogrande": { + "acc": 0.5019731649565904, + "acc_stderr": 0.014052376259225629 + }, + "storycloze_2016": { + "acc": 0.5606627471940139, + "acc_stderr": 0.01147701798230878 + }, + "boolq": { + "acc": 0.5740061162079511, + "acc_stderr": 0.008648732832949143 + }, + "arc_easy": { + "acc": 0.35774410774410775, + "acc_stderr": 0.00983577275734336, + "acc_norm": 0.3409090909090909, + "acc_norm_stderr": 0.009726579593424019 + }, + "arc_challenge": { + "acc": 0.17406143344709898, + "acc_stderr": 0.01108017712948222, + "acc_norm": 0.2226962457337884, + "acc_norm_stderr": 0.012158314774829931 + }, + "sciq": { + "acc": 0.646, + "acc_stderr": 0.015129868238451773, + "acc_norm": 0.597, + "acc_norm_stderr": 0.01551875741906653 + }, + "piqa": { + "acc": 0.6240478781284005, + "acc_stderr": 0.011301098166895732, + "acc_norm": 0.6169749727965179, + "acc_norm_stderr": 0.011342081709082847 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_4.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..5b09beffcac5ab15de7a7b75ba89eebe4bc7520e --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.342,0.015008706182121731,0 +anli_r2,acc,0.341,0.014998131348402706,0 +anli_r3,acc,0.3283333333333333,0.013562032919529026,0 +arc_challenge,acc,0.17320819112627986,0.01105869418328034,0 +arc_challenge,acc_norm,0.21245733788395904,0.011953482906582947,0 +arc_easy,acc,0.3581649831649832,0.009838331651451848,0 +arc_easy,acc_norm,0.3404882154882155,0.009723676813825868,0 +boolq,acc,0.5972477064220183,0.008578054401368405,1 +cb,acc,0.375,0.06527912098338669,1 +cb,f1,0.25089094796863864,,1 +copa,acc,0.58,0.049604496374885836,0 +hellaswag,acc,0.27763393746265685,0.004469165728600334,0 +hellaswag,acc_norm,0.29028082055367455,0.0045296428285464055,0 +piqa,acc,0.6175190424374319,0.011339019654272345,0 +piqa,acc_norm,0.6169749727965179,0.01134208170908285,0 +rte,acc,0.5090252707581228,0.030091559826331334,0 +sciq,acc,0.643,0.01515852172148677,0 +sciq,acc_norm,0.615,0.01539519444541081,0 +storycloze_2016,acc,0.5595938001068947,0.011480012367207821,0 +winogrande,acc,0.4996053670086819,0.014052481306049516,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_4.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_4.json new file mode 100644 index 0000000000000000000000000000000000000000..113dbcbdf1fac7b26c0180487dbc91e3e80ff9f3 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.342, + "acc_stderr": 0.015008706182121731 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402706 + }, + "anli_r3": { + "acc": 0.3283333333333333, + "acc_stderr": 0.013562032919529026 + }, + "cb": { + "acc": 0.375, + "acc_stderr": 0.06527912098338669, + "f1": 0.25089094796863864 + }, + "copa": { + "acc": 0.58, + "acc_stderr": 0.049604496374885836 + }, + "hellaswag": { + "acc": 0.27763393746265685, + "acc_stderr": 0.004469165728600334, + "acc_norm": 0.29028082055367455, + "acc_norm_stderr": 0.0045296428285464055 + }, + "rte": { + "acc": 0.5090252707581228, + "acc_stderr": 0.030091559826331334 + }, + "winogrande": { + "acc": 0.4996053670086819, + "acc_stderr": 0.014052481306049516 + }, + "storycloze_2016": { + "acc": 0.5595938001068947, + "acc_stderr": 0.011480012367207821 + }, + "boolq": { + "acc": 0.5972477064220183, + "acc_stderr": 0.008578054401368405 + }, + "arc_easy": { + "acc": 0.3581649831649832, + "acc_stderr": 0.009838331651451848, + "acc_norm": 0.3404882154882155, + "acc_norm_stderr": 0.009723676813825868 + }, + "arc_challenge": { + "acc": 0.17320819112627986, + "acc_stderr": 0.01105869418328034, + "acc_norm": 0.21245733788395904, + "acc_norm_stderr": 0.011953482906582947 + }, + "sciq": { + "acc": 0.643, + "acc_stderr": 0.01515852172148677, + "acc_norm": 0.615, + "acc_norm_stderr": 0.01539519444541081 + }, + "piqa": { + "acc": 0.6175190424374319, + "acc_stderr": 0.011339019654272345, + "acc_norm": 0.6169749727965179, + "acc_norm_stderr": 0.01134208170908285 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_5.csv b/619m2b71b5/evaluation/rankeval/619m2b71b5_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..8e0897afcba8086a1adcf29c8836fd97e798dad6 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.328,0.014853842487270333,0 +anli_r2,acc,0.336,0.014944140233795027,0 +anli_r3,acc,0.3425,0.013704669762934723,0 +arc_challenge,acc,0.17406143344709898,0.011080177129482222,0 +arc_challenge,acc_norm,0.21416382252559726,0.011988383205966487,0 +arc_easy,acc,0.36784511784511786,0.009894923464455183,0 +arc_easy,acc_norm,0.3463804713804714,0.009763542075695741,0 +boolq,acc,0.5856269113149847,0.00861586377642113,1 +cb,acc,0.5357142857142857,0.0672477765493766,1 +cb,f1,0.3734827264239029,,1 +copa,acc,0.58,0.049604496374885836,0 +hellaswag,acc,0.2773351921927903,0.004467684132772413,0 +hellaswag,acc_norm,0.2922724556861183,0.004538773493746564,0 +piqa,acc,0.6207834602829162,0.011320331012905067,0 +piqa,acc_norm,0.6164309031556039,0.011345128734116278,0 +rte,acc,0.5054151624548736,0.030094698123239966,0 +sciq,acc,0.652,0.015070604603768408,0 +sciq,acc_norm,0.616,0.015387682761897071,0 +storycloze_2016,acc,0.5595938001068947,0.011480012367207817,0 +winogrande,acc,0.5209155485398579,0.014040185494212945,0 diff --git a/619m2b71b5/evaluation/rankeval/619m2b71b5_5.json b/619m2b71b5/evaluation/rankeval/619m2b71b5_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ca021b0700294dd391464d99e767e882b1b2a3f0 --- /dev/null +++ b/619m2b71b5/evaluation/rankeval/619m2b71b5_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.328, + "acc_stderr": 0.014853842487270333 + }, + "anli_r2": { + "acc": 0.336, + "acc_stderr": 0.014944140233795027 + }, + "anli_r3": { + "acc": 0.3425, + "acc_stderr": 0.013704669762934723 + }, + "cb": { + "acc": 0.5357142857142857, + "acc_stderr": 0.0672477765493766, + "f1": 0.3734827264239029 + }, + "copa": { + "acc": 0.58, + "acc_stderr": 0.049604496374885836 + }, + "hellaswag": { + "acc": 0.2773351921927903, + "acc_stderr": 0.004467684132772413, + "acc_norm": 0.2922724556861183, + "acc_norm_stderr": 0.004538773493746564 + }, + "rte": { + "acc": 0.5054151624548736, + "acc_stderr": 0.030094698123239966 + }, + "winogrande": { + "acc": 0.5209155485398579, + "acc_stderr": 0.014040185494212945 + }, + "storycloze_2016": { + "acc": 0.5595938001068947, + "acc_stderr": 0.011480012367207817 + }, + "boolq": { + "acc": 0.5856269113149847, + "acc_stderr": 0.00861586377642113 + }, + "arc_easy": { + "acc": 0.36784511784511786, + "acc_stderr": 0.009894923464455183, + "acc_norm": 0.3463804713804714, + "acc_norm_stderr": 0.009763542075695741 + }, + "arc_challenge": { + "acc": 0.17406143344709898, + "acc_stderr": 0.011080177129482222, + "acc_norm": 0.21416382252559726, + "acc_norm_stderr": 0.011988383205966487 + }, + "sciq": { + "acc": 0.652, + "acc_stderr": 0.015070604603768408, + "acc_norm": 0.616, + "acc_norm_stderr": 0.015387682761897071 + }, + "piqa": { + "acc": 0.6207834602829162, + "acc_stderr": 0.011320331012905067, + "acc_norm": 0.6164309031556039, + "acc_norm_stderr": 0.011345128734116278 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7626021dc4e0f644b1cd7fdd779f65177f08e726 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89e870d773b4e4502bebd418fabc9589e145712870112b9c48b20f90f358a489 +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75d0b248c955b99b71b95951f1d30f0cc6741711 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e117162b118023188ba0851bbd0dfef2cae9129753292bef674928f9060d70ee +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41a4c3844d5f5fa8ad7f1d1ea0f3062e0fd934b2 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664c3eeb902b8d5c0ab1b1ecebad237992c2e75dbe13d8e7eab093bdbb78602f +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c81549039a6a4d4c19b16b3c8a8ac578020c05e4 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79eff52403bac3f5828ce69d969e66423443130069603878e3818899c1ed3883 +size 116013602 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..673ac0e96861158d3bc00804708ab212ee37591f --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e748b40fe79aa64cc9ed4039d473d33d3623979ed6cbfdb94cc664118f86ce51 +size 116013858 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c45d080f4d0f5d79b31cf622d391718fe2e47839 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a5145af6477bec93ae13dd032d5d51ca530ea7ed83121b89c74c6c96b9521df +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2195e9aede4fa0949e4d41b5b6bcbe517e70172b --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8d4c03a3e5154a75452058b129165bb0bc41b1d7a23db7fbf0b7ce55601c256 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55dc842e7bada401b1e076fe7dbaba8df37d474a --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba9a787fe3700b7f8303fe25d70c653b792a72dd7d4e23b985d2bd596817a67 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dde147170e789f856c64d75e2aef1e2f21f5797 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9199e85a495b887b063ecd6bb73ea5975a9a709f17af0a22332e5e0f1d68c7 +size 116013602 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..239bbc5f0469a911300a0af2ee02d43367843373 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708ee508dd776001e59011f108c99b544591703e01bc89670b823ebfb6bea7a5 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22c07ede528ef53e9fb6def03fc0f3940557aa24 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc375ed6bb41bdd15f58d608e14aa66ee4cb91e78be24a34f2354b7bc9922d03 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88a8872055cfb4e8bc893c3d726e0d26c5c27e8e --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3890c3a6017c223d6b5fff3b74e71bac8bc8a9a16f81950be2e97d4dd045d7 +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70296a80a661bde484c4cae819c92df0e0144330 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36e6c9abe793bbe10fda87bbb0c1ecd7d5b899dee4998128da69abef38c6021 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b3f20704e50045abbfa8f60266639edde05f929 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c289f63dd8a0d815641d559b9a48d980d1ce744aafc4a6028e3a5bd3554b9f1 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fc4214e1eb9b4015a8b5fa3e220eb7c20dddcef --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:991fc2e1a665fc6bcd8189bdd00618efd54d77178a411af84ed3e48befd47203 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..502f9f5a0f53ad8a5eaf6fdddde9d53498577fa5 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3807df661b9d48cec31d5196bc5e5f92e3c6b722db779e403b734c7e6512fd2b +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2eee5ee512d1819544625ac91f3c7e1225d0348 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196403d43cd6a1dcb08c8c1e1f321c681fc3710ee501e05f536980995eb698b2 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29bca3e940cf10d53355977842b631c4667a8418 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f453399477b903b8fd110b2e982823fc60336549a032d08df564e04c133d82df +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2699e37113f65a85af6d9291f6651dd837443bb3 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f84dfd05ea720155ba6e3bfbee4b2e1b32e7bc7eaec6c51897e185a2188866 +size 116013858 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d701467c9af47b0b47eb8cc5367d5bb3dc0e946 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0208c75146d66d32715521acc1e765fb4b6bae08d25cbb235f79b1f00e389ec9 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..276ba841fe9d3b770b3039a33039a8c050ab03a3 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24b59d9f41e061a196ad12a80a18b85bfa17d758ebece4c057eadb104597a79a +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b6db2307fa2724c29bf32ff7deaa1513fafb63d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91483e0e2db114d33044721c166c11528c14fce8649dda6b3bf6cdf957d5bd50 +size 116013602 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6c5e2450853d13d054cc14f6d82bc5f43d06eca --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35640de9900ee2ca75990ff6b7005ad194cbff8658cbac81b695c20740429bc2 +size 116013655 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aff2fedf86c4c5055e01ce1765a584795bedb09b --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8acb3ccd96c36ce8ec1ce60c1066e3d58403822c2f2e77818efca6d6d233d8f +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b507be11b949593da92d62678ecd6eca03d4a3d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f986373f11fb12406ae33921080fb61de9a4660578a4e4c675819ee8f97b0952 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02d56c7494e8794a1adefbfd039d4eee7e83ce78 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd6178e3399e41ecdc3bafa2ad5baf2b180b78392e3c6aeb10dab6752a643f19 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87c80f076c32104410a767339bbfe3e9d4b1eaeb --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4226b07aab898fd5da6b9589ab9d9283be9410df96d373bce342fcbf2fff8f38 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..048083f5e1810e76b0721400bc8bc5d9f57a8d98 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26a33c618a80efbaffc0bee2f3c3f77d5dc233af1d4943b3e37d31ae4b9cb7a +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dffcfbd98d4a5f672b154a574cf661d6cfe1905d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b28c802f2ec34d91bf81582216567796d7b189965cff7c188bcfa05418a4083 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5ec7dd71c37c6dfe44aac6ead23d999d9275a73 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a2443fd47ba92535a2e4c88125fd16c7cd568f375107688e9f4ecf07b550be8 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a83de40e2451e9fc7ea9ae921f2bad0891c3e37 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c61f13054af36c51eee7470a479847fb47ba10825d63fc0cb7082369d5f3a829 +size 116013858 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15afbb0cee7d96a2c2405d81b6ca6ce3c460938b --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5e77d9e6c65784fb5c7ec2b9d62524603d397869372f227a232b7c4278c9765 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50d1e72b5995d79acbe73dd4edd818b8d2ace372 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:106a12758aee5a441e5f41cfcac017d2545986e832077be02c1621eb50eaac80 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..630faacb4733d92fa6c0d16f33c9c4da892f8600 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:233539fe9f8046a710b2e7b5acd1575b653fb00fffd8d717051e6200b4081fab +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ff6db1e4a7c47c769312d24d5a32e570ac2c753 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6640ba0427346e7db452f40d6906746cef1897fc000f3a88cde015c913d5a95d +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..022c751cd878f5eacf59ca264bbf41a9f972f112 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b170c2984dfac2ec69106874777ec0bca29d9252e1f67083b2bdffa26b61ba88 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1482e2028bab8ba04f7dfcad9555c4127da0139 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:485bb1679a3ae3d19ccd98af5edd70dbe29a396c7438e31de57d7a577d815e72 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10e3795ca659619e428b358b688cf7763582f29f --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dc1a1766393be978d71a6296476097ef0f6f1e2857b574feeb48acfb0d1bb81 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d81aa7c8e99bc9420b8024324cece5ae62f5458d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:defd83e59be5ae87df31d345d618019941f1b929148a1304f66e9b72f25a0a36 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7838df3269b6f9688f29d4746d1e28108f9e1890 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a340d5249194e299cb902b98f9f0c0dd880b0ecf0a7ba54be0e0f9d03079ffb +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40f4001d17ac3acb053d014a568c109b0dada6c2 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9967b31fd38aac15bbf9292a14c8f3d55c497fa35ecf57a874ff28bd70e9358 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9b381c0cfcc818f648143ab8503a98d8591018b --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:547f47e33c388cbe2dc15d9fd95b1bbba524ed4ec8f47cecc3a9faf983a3febc +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f39be847c62dcdfb715e74f0c29e06926c23132 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9a9815aab89035aced6c0cdc06d644510f55105ba9a8a7ec35b2d72617dd55 +size 116013922 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17eadbc9fb2a964a6b247c43a2f32223dd9b5b28 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:807ccf536e6d5463367266bcba5158497aebda65ba4ef9790e204c5af4920d31 +size 116013538 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1369ddec996c121e9306483da70a3ce3329172ac --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9282d83141149ebfec8e9404775fe7a1653f9acdcea422166df736bd18d0d8a +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f95a52ea4ce07b1d20544be9a9c4d8db490489d4 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad81991aaada6cb9fdd10ea1a97cd95f3bc7a05815cdb116c5a0b9b93d9c6ffb +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae5742430726ae719dd0d3fdef59ac391c0d3cea --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fdc63e08ae4e8ac1e0331274465acb33bbd51259c2820c2961c43cf49d1af3a +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21a04bcecf305e3242ce0fce7a45a6f4e7483025 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8845548de5b9d2cf95da509e61215a39e924d0e468e81b96d885e06f10ea071b +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e16c87c302dc00baf0d769624da1fa76e12c7fba --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:572972959dee72a946509241fd25369519b467727a37ab29db905b2eff84e4eb +size 116013922 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5caa4cb6f8c3e88aab95a357a3181749a6f4767b --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c416edf7dd292ae1e9d40438d592f188c7a0202ea572a4dfe62018ea87da4e +size 116013602 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41885d016faba336aec5fbca74f40ed5d564d9b9 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c854d5aa211bf8f4cbe021ea7207c602f744b7a3111f5a40462c8fbba456aa2e +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..420325ef121974b28be2272310d42a2a87502a4d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cff93c4185d06f14522f656a566ac4093d53c6cf47f1f218d31717e0d157568 +size 116013602 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a7a8f098205f4e756f583e8a81194105276c725 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6195e904a8975a1b0f3ad531786aeaa8f4fed69c6ada5ed7997401de19dad785 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c83ce5f0f8b992b4ef31ec4743cab98b5cc5668a --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0b96c3d27252a35274b01813242c0b4f282d02d8352c0a9fe3edbe167518471 +size 116013858 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33509236512c957bdf0800f121fafb6f6734bcbb --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3e83c932e4e9054ea6e36cd402c135c43eec25cfd095e13988e430b1e16466 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f56a29eaaabbcb4edac029dc58b5ca6ab03caa9a --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786235ce732a84d82de1ebce76f5f6dc6e86d5cfea0983f3b511ccb49a549cf7 +size 116013655 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..562e30fef2fcc925a59dfe5e5811ad415a0f0ff1 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:934a4ffacd28f574d3d35b8bdbc67591c4520b3c4e44b7929f1ef693fedd12b1 +size 116013730 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8a357226bfca2eab8eb3cbe09d5fadfcae9262d --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:298d510e6e86710b176a546e7ea4880ac365c151b9377c3be8068e621af43912 +size 116013794 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81d7eb4a96fb605dadf3202d565e85c92c7285f9 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e9e2c04c6caf417922909ef670ef928a0b7411d32366c2515cc86d49971da4a +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..157aa5e512183bf8d2aedc419981c5d1e00b022c --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d508f231f4a84b736798714c7fe9c40e82156b7a0560538d669e426deeb0a4 +size 116013666 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..204c4232d441c6666521ec36a25905054edf2a27 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c9d79a77df4d1837b5a95b5f1919757d6c34586710862ce79b184193d72aa9 +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce95cee58db863b6e9538aaf4e2377ad00ff9afa --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a73e7a4a4afe55b1ab788b0c43035bdf0d2a274eae648c2dd7dbbad798f58b +size 116013719 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3c155cc9b2c6b5438e888b8f36b03eb5b6a82f1 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28bfc6f8f938d908ba4a69c9772d846a3fd2a6af371b7310489870545e54e4e3 +size 116013655 diff --git a/619m2b71b5/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m2b71b5/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92bde4d26722ad53c16cd9504a7c3de6c4d4c5f8 --- /dev/null +++ b/619m2b71b5/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b4665f4642572170a12d6097813319a41d425672947e321702857b83bffb94 +size 116013591 diff --git a/619m2b71b5/global_step5111/layer_01-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69ded51188dd2bf01abbae35ca56acc1a9ca380d --- /dev/null +++ b/619m2b71b5/global_step5111/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97395684fe401975ba88e209e57d1d0acdbb92417a5f81aa55bb3b78b27c92b9 +size 160826627 diff --git a/619m2b71b5/global_step5111/layer_03-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a748c0f50202c408773a80b65a5cbf0503403c65 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ffb4200fb5169036398de51e4db8e98de9a3433d1e82b565084708e47b53424 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_04-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2b2749145a6cf46d57ec0c72a42f3d286199d6c --- /dev/null +++ b/619m2b71b5/global_step5111/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ed732f8d3e9f54f8119d2da58889fcfb177d23a6012fa81dbc369f0ad6a6173 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_05-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08d9a8c920c6916f4d89513993e8f45d4ad2a1d0 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bdcc129a6f2b6e35ed176ad5662205f801c26fe83137a5ba09c289fcca76f04 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_06-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a23ac094134a3c936f9d523cd110f4a3d6b76c61 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:668ee86bba53688dd7b91dd9dd6bab98c2f03efda629aa8975584be620fbbc96 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_07-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92458b898fe41dcd6c060e864f45e9f1b6bc9eaf --- /dev/null +++ b/619m2b71b5/global_step5111/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999fd641294664ed9e5bba9220d4ad27b3aec4032395d6745007342c5cdb0020 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_08-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dc5944805b4817a539765eca0c62653129d501d --- /dev/null +++ b/619m2b71b5/global_step5111/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ea48140db286ab668eb1da1933d7bb3fb41f68a6d8b34de3b27500bff14d88 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_09-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed707fb9985bdc7ceb1e147afd58aaf15e60cc70 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ee7a1eaca02d7a84b39afff0768fd252455774e3c447966b9621e6ff0da351 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_10-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..042672be09a371b9e7f4e20c43ffc680b142c2e6 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b47416c01eedb03a5197adb3f53cb479ba6d4f6fc8aa94498409a1049ae41205 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_11-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdb58f07b09fe1166ed21ac3701f2c94cb54fb65 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c265748db15f975a4b402f19275bc6df535d190f5166579012339188153f4119 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_12-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7a1e493c36d0acda4c79b3028d472b4910be4bb --- /dev/null +++ b/619m2b71b5/global_step5111/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e9924a1afe645d76fb59d440a14487b0c36fa8f8762babbb7bff751c3615dd +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_13-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97e88900c2985cd5888d3c87cfe8384afddfd1fa --- /dev/null +++ b/619m2b71b5/global_step5111/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e139c431f30c1b22273155aa3ed9c544b698334cb8876c0e033beb89e144e3c6 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_14-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..946a045b3763ff8e8b92fd2228c8f481d180f223 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5174a4934562aa3d59cecff916d24c8ded947904093e705a17d1a7b94074ef +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_15-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..844c35ec5fadcaf0278ac43e70f2fcb11b7b107b --- /dev/null +++ b/619m2b71b5/global_step5111/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3550f71f13a3d73eaecc871d2e82aa7010d40141707fe7f0c38743d6034a8280 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_16-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5d15c97326eeb5e94b56d9fe32d84c19827d645 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acd88ee1e3f455a4451b1b35940a30aa0995fb4f075bc080941228ddfeb68a65 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_17-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4e5555b51119642719cff91264d91af53d6ba19 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4039ca254b7d07c922f9bac1aa5c6ca30fdc523bcff098a8948e71037f2cc779 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_18-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5501636f2b8a769280af0139271769da3e10c2f --- /dev/null +++ b/619m2b71b5/global_step5111/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d92ea60cca347e54237e0c881b729c65cd7b5e1818fe4777c25674c6c72e3a3 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_19-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efdf7469710886a6faff93bf0d4ddb6502c75ece --- /dev/null +++ b/619m2b71b5/global_step5111/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a22ab2b28411262ef2222dc06d911bb49d62d94bf99ad782d4d8104823f413e +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_20-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14ff816abdfd2857eebc32bec7af7ba418821b91 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1f099a8b03482bd663e5eab461209ca0ad806e1c54103650a8b7955f0bedf6f +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_21-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad8a5969fd1f6588f38a86b10bba81cc3008102e --- /dev/null +++ b/619m2b71b5/global_step5111/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65fde3b593deadd0dd59f25630b773253d35155aa4f966af4c7aed1b2541a01 +size 56667395 diff --git a/619m2b71b5/global_step5111/layer_23-model_00-model_states.pt b/619m2b71b5/global_step5111/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01b3e6153ce021bc380e22798340c0bee9b61db4 --- /dev/null +++ b/619m2b71b5/global_step5111/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23847a0d74b824c5728d9bd02e169fdec237cee1bea3178c95a83ec832807a1b +size 7363 diff --git a/619m2b71b5/global_step5111/mp_rank_00_model_states.pt b/619m2b71b5/global_step5111/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dba0b040167be2143f3e38b7c6961a3b88334f4 --- /dev/null +++ b/619m2b71b5/global_step5111/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c465823a3bd543e7975f8d82238d216e7027dc427a55835b598c934f228d0c +size 38451 diff --git a/619m2b71b5/logs/2820895.err b/619m2b71b5/logs/2820895.err new file mode 100644 index 0000000000000000000000000000000000000000..aaafe33b2257f5178e0b3add382e19fb114ccc0b --- /dev/null +++ b/619m2b71b5/logs/2820895.err @@ -0,0 +1,1114 @@ +0: 2023-02-09 22:47:59.830402: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.830412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.830416: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.830425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.837899: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.837913: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.837909: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.842252: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.842257: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.842256: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.842271: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.843302: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.845872: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.845880: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.845877: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.845894: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.846539: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.846570: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.846581: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.847797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.850064: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.855264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.862707: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.866870: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.875395: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.878079: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.883084: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.887960: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.893505: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.900809: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.903842: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.908767: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.916848: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.924420: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:47:59.927429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.929756: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.935685: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938821: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938831: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938835: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938839: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.938840: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.941189: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.951372: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.952178: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.962398: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986543: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986550: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986554: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986548: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986540: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986539: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986563: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.986555: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090786: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090783: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090793: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090800: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090791: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090786: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090796: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:48:00.090806: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:01.739658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.739663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:01.740159: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740161: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740165: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740169: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740172: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740174: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740172: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:01.740177: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.745817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745822: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.745823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:01.746044: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746047: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746049: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746051: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746051: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:01.746055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.795540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795576: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795598: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.795606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:01.796010: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796013: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796041: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796048: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796053: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796068: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796070: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:01.796072: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:01.856963: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856964: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856970: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856971: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856973: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856974: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856977: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:01.856980: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017387: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017396: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:02.017810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017816: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017814: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017819: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017820: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017820: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017822: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:02.017826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400403: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400419: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400429: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.400776: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400783: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400790: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400799: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400795: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400797: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400802: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.400812: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.480555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.480566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:02.481024: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481025: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481028: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481030: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481031: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481033: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481036: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:02.481038: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:02.493947: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493952: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493953: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493956: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493963: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493970: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493966: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:02.493988: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:13.751872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.751893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.753648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.753974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.753979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.753987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.753984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.753991: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.753988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.753988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.753989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.754001: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.754003: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.754004: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.754005: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.754067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.754070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.754079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.754082: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755600: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755591: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.755604: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755607: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755610: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755609: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755611: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.755614: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.757441: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.757694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-09 22:48:13.757469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757470: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 22:48:13.757698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 22:48:13.757703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.757481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 22:48:13.757711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.757706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.757706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.757714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.757715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759329: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759348: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759709: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759724: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759729: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759732: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759734: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759733: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759781: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.759787: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.759795: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.773854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.773868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776061: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776066: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.776283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.776287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.798747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798760: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798771: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.798785: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759369: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759369: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759371: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.759392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.759415: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810467: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-09 22:48:13.810572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810483: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.810483: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.810484: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.810488: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.810488: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-09 22:48:13.810571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-09 22:48:13.810578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.810584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.810584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-09 22:48:13.810555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.810570: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.810572: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.810583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-09 22:48:13.810574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.810583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.810588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812832: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812832: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812834: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812835: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812835: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812879: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.812887: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.812891: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.801475: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801475: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801479: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801481: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801483: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801483: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801485: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.801486: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +4: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +4: Successfully preprocessed all matching files. +7: Successfully preprocessed all matching files. +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +3: Building extension module utils... +3: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Loading extension module utils... +7: Loading extension module utils... +5: Loading extension module utils... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: +6: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +6: Loading extension module utils... +1: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +3: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... +3: +3: +3: +3: +3: Loading extension module utils... +3: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +5: Loading extension module utils...Loading extension module utils... +5: Loading extension module utils... +5: +5: Loading extension module utils...Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: +1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +5: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +0: Loading extension module utils...Loading extension module utils... +0: +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils...Loading extension module utils...Loading extension module utils... +7: +7: +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils...Loading extension module utils... +6: +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: +7: Loading extension module utils...Loading extension module utils...Loading extension module utils... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m2b71b5/logs/2820895.out b/619m2b71b5/logs/2820895.out new file mode 100644 index 0000000000000000000000000000000000000000..5dc0e5c9b2f337756348c16915cffdda86bff4b7 --- /dev/null +++ b/619m2b71b5/logs/2820895.out @@ -0,0 +1,6682 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m2b71b5val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_619m2b71b5val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m2b71b5 --load checkpoints_619m2b71b5 --train-weighted-split-paths-path train2b7.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2820895.json --zero-stage 0 +START 2820895: Thu 09 Feb 2023 10:47:30 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 43.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 44.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 49.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 40.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 39.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 36.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 42.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 43.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 41.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 41.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 38.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +2: Launching on nid005034 (2/8), master nid005032 port 9999, GPUs 8, CUDA: True +3: Launching on nid005035 (3/8), master nid005032 port 9999, GPUs 8, CUDA: True +6: Launching on nid005038 (6/8), master nid005032 port 9999, GPUs 8, CUDA: True +5: Launching on nid005037 (5/8), master nid005032 port 9999, GPUs 8, CUDA: True +4: Launching on nid005036 (4/8), master nid005032 port 9999, GPUs 8, CUDA: True +7: Launching on nid005039 (7/8), master nid005032 port 9999, GPUs 8, CUDA: True +1: Launching on nid005033 (1/8), master nid005032 port 9999, GPUs 8, CUDA: True +0: Launching on nid005032 (0/8), master nid005032 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/2820895.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... True +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m2b71b5val +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m2b71b5 +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m2b71b5 +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m2b71b5val +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +7: > setting tensorboard ... +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-09 22:48:56,897] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.098 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 102 +0: [1/1] c++ scaled_masked_softmax_hip.cuda.o scaled_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: ninja: no work to do. +0: >>> done with compiling and loading fused kernels. Compilation time: 36.029 seconds +0: time to initialize megatron (seconds): 95.960 +0: [after megatron is initialized] datetime: 2023-02-09 22:49:35 +0: building GPT model ... +0: [2023-02-09 22:49:36,095] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-09 22:49:36,096] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-09 22:49:36,096] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.2 GB, percent = 6.0% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-09 22:49:38,092] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-09 22:49:38,366] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-09 22:49:38,367] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-09 22:49:38,367] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.24 GB, percent = 6.0% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-09 22:49:38,369] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-09 22:49:51,801] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-09 22:49:51,802] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-09 22:49:51,802] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-09 22:49:51,820] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-09 22:49:51,820] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-09 22:49:51,940] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-09 22:49:51,941] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 22:49:51,941] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.92 GB, percent = 6.1% +3: ninja: no work to do. +3: Time to load utils op: 0.18642854690551758 seconds +5: Time to load utils op: 0.20913028717041016 seconds +7: Time to load utils op: 0.21003341674804688 seconds +0: Time to load utils op: 0.11120295524597168 seconds +0: [2023-02-09 22:49:52,163] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-09 22:49:52,164] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 22:49:52,164] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.93 GB, percent = 6.1% +0: ninja: no work to do. +0: Time to load utils op: 0.2356255054473877 seconds +2: Time to load utils op: 0.11153531074523926 secondsTime to load utils op: 0.11156582832336426 seconds +2: +2: Time to load utils op: 0.11158132553100586 seconds +2: Time to load utils op: 0.11154866218566895 seconds +2: Time to load utils op: 0.11156153678894043 seconds +2: Time to load utils op: 0.11156582832336426 secondsTime to load utils op: 0.11155104637145996 secondsTime to load utils op: 0.1115727424621582 seconds +2: +2: +6: Time to load utils op: 0.11186599731445312 secondsTime to load utils op: 0.11188387870788574 seconds +6: +6: Time to load utils op: 0.11188268661499023 seconds +6: Time to load utils op: 0.1119077205657959 secondsTime to load utils op: 0.11192679405212402 seconds +6: +6: Time to load utils op: 0.11190199851989746 seconds +6: Time to load utils op: 0.11193561553955078 seconds +6: Time to load utils op: 0.11195945739746094 seconds +1: Time to load utils op: 0.11220026016235352 seconds +1: Time to load utils op: 0.11221766471862793 seconds +1: Time to load utils op: 0.1121513843536377 seconds +1: Time to load utils op: 0.11220884323120117 seconds +1: Time to load utils op: 0.11216855049133301 seconds +1: Time to load utils op: 0.11220979690551758 seconds +1: Time to load utils op: 0.11217522621154785 seconds +1: Time to load utils op: 0.1122126579284668 seconds +0: [2023-02-09 22:49:52,339] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-09 22:49:52,340] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 22:49:52,340] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.93 GB, percent = 6.1% +3: Time to load utils op: 0.0005826950073242188 seconds +5: Time to load utils op: 0.0004487037658691406 seconds +7: Time to load utils op: 0.0004343986511230469 seconds +0: Time to load utils op: 0.0007393360137939453 seconds +0: Time to load utils op: 0.30297088623046875 seconds +0: Time to load utils op: 0.3018002510070801 seconds +3: Time to load utils op: 0.3028564453125 seconds +3: Time to load utils op: 0.3036034107208252 seconds +3: Time to load utils op: 0.3029909133911133 seconds +3: Time to load utils op: 0.3030226230621338 seconds +3: Time to load utils op: 0.3029441833496094 seconds +3: Time to load utils op: 0.3040587902069092 seconds +3: Time to load utils op: 0.30281639099121094 seconds +1: Time to load utils op: 0.0009899139404296875 seconds +1: Time to load utils op: 0.0010979175567626953 seconds +1: Time to load utils op: 0.0012717247009277344 seconds +1: Time to load utils op: 0.001336812973022461 seconds +1: Time to load utils op: 0.0012946128845214844 secondsTime to load utils op: 0.0013666152954101562 seconds +1: +1: Time to load utils op: 0.0012180805206298828 seconds +1: Time to load utils op: 0.0014104843139648438 seconds +5: Time to load utils op: 0.30420899391174316 seconds +5: Time to load utils op: 0.3039398193359375 seconds +5: Time to load utils op: 0.30467796325683594 seconds +5: Time to load utils op: 0.30409693717956543 seconds +5: Time to load utils op: 0.3043363094329834 seconds +5: Time to load utils op: 0.30434322357177734 seconds +5: Time to load utils op: 0.3049733638763428 seconds +0: Time to load utils op: 0.30269384384155273 seconds +0: Time to load utils op: 0.3027384281158447 seconds +0: Time to load utils op: 0.30236268043518066 seconds +0: Time to load utils op: 0.30298709869384766 seconds +6: Time to load utils op: 0.0008933544158935547 seconds +6: Time to load utils op: 0.0007646083831787109 seconds +7: Time to load utils op: 0.3051176071166992 secondsTime to load utils op: 0.3051607608795166 seconds +7: +7: Time to load utils op: 0.30518555641174316 seconds +7: Time to load utils op: 0.3051481246948242 seconds +6: Time to load utils op: 0.0013697147369384766 seconds +7: Time to load utils op: 0.30529236793518066 seconds +7: Time to load utils op: 0.3052103519439697 seconds +6: Time to load utils op: 0.001287221908569336 seconds +6: Time to load utils op: 0.0014252662658691406 seconds +6: Time to load utils op: 0.0010721683502197266 seconds +7: Time to load utils op: 0.30542826652526855 seconds +6: Time to load utils op: 0.0011820793151855469 seconds +6: Time to load utils op: 0.0013086795806884766 seconds +4: Time to load utils op: 0.31238293647766113 secondsTime to load utils op: 0.3119804859161377 seconds +4: Time to load utils op: 0.31206583976745605 seconds +4: +4: Time to load utils op: 0.31211352348327637 secondsTime to load utils op: 0.3125276565551758 secondsTime to load utils op: 0.3119328022003174 seconds +4: +4: +4: Time to load utils op: 0.3120734691619873 seconds +4: Time to load utils op: 0.3119640350341797 seconds +2: Time to load utils op: 0.0008153915405273438 seconds +2: Time to load utils op: 0.0008063316345214844 seconds +2: Time to load utils op: 0.0008015632629394531 seconds +2: Time to load utils op: 0.0008394718170166016 seconds +2: Time to load utils op: 0.0008425712585449219 seconds +2: Time to load utils op: 0.0010149478912353516 seconds +2: Time to load utils op: 0.0009844303131103516 seconds +2: Time to load utils op: 0.0010285377502441406 seconds +0: Time to load utils op: 0.00039505958557128906 seconds +0: Time to load utils op: 0.0003757476806640625 seconds +3: Time to load utils op: 0.00035858154296875 seconds +3: Time to load utils op: 0.00034332275390625 seconds +3: Time to load utils op: 0.0003533363342285156 seconds +3: Time to load utils op: 0.0003638267517089844 seconds +3: Time to load utils op: 0.0003540515899658203 seconds +3: Time to load utils op: 0.00034809112548828125 seconds +3: Time to load utils op: 0.00034356117248535156 seconds +5: Time to load utils op: 0.0003476142883300781 seconds +5: Time to load utils op: 0.00034308433532714844 seconds +5: Time to load utils op: 0.0003323554992675781 seconds +5: Time to load utils op: 0.0003209114074707031 seconds +5: Time to load utils op: 0.0003616809844970703 seconds +5: Time to load utils op: 0.0003609657287597656 seconds +5: Time to load utils op: 0.00036787986755371094 seconds +0: Time to load utils op: 0.0004019737243652344 seconds +0: Time to load utils op: 0.0003943443298339844 seconds +0: Time to load utils op: 0.00040435791015625 seconds +7: Time to load utils op: 0.0003840923309326172 seconds +7: Time to load utils op: 0.0003666877746582031 seconds +0: Time to load utils op: 0.0003781318664550781 seconds +7: Time to load utils op: 0.0005223751068115234 secondsTime to load utils op: 0.0005002021789550781 secondsTime to load utils op: 0.0005033016204833984 seconds +7: +7: +7: Time to load utils op: 0.00036644935607910156 seconds +7: Time to load utils op: 0.0004894733428955078 seconds +0: [2023-02-09 22:49:52,452] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-09 22:49:52,453] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 22:49:52,453] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +4: Time to load utils op: 0.0008854866027832031 seconds +4: Time to load utils op: 0.0010600090026855469 seconds +4: Time to load utils op: 0.0011277198791503906 seconds +4: Time to load utils op: 0.0013051033020019531 seconds +4: Time to load utils op: 0.0012707710266113281 seconds +4: Time to load utils op: 0.001277923583984375 seconds +4: Time to load utils op: 0.0013477802276611328 seconds +4: Time to load utils op: 0.0013496875762939453 seconds +0: [2023-02-09 22:49:52,559] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-09 22:49:52,560] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:52,560] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:52,664] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-09 22:49:52,665] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:52,665] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:52,771] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-09 22:49:52,772] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:52,772] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:52,875] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-09 22:49:52,876] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:52,876] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:52,985] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-09 22:49:52,985] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:52,986] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:53,089] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-09 22:49:53,090] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:53,090] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.07 GB, percent = 6.2% +0: [2023-02-09 22:49:53,090] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-09 22:49:53,090] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-09 22:49:53,090] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-09 22:49:53,090] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-09 22:49:53,091] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-09 22:49:53,092] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-09 22:49:53,093] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-09 22:49:53,093] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.00041794776916503906 seconds +0: [2023-02-09 22:49:53,093] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-09 22:49:53,176] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:53,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:53,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:53,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:53,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:53,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:53,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:53,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:53,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:53,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:53,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:53,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:53,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:53,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:53,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:53,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:53,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:53,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:53,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:53,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:53,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:53,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:53,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:53,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:53,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:53,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:53,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:53,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:53,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:54,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:54,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:54,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:54,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:54,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:54,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:54,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:54,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:54,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:54,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:54,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:54,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:54,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:54,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:54,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:54,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:54,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:54,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:54,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:54,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:54,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:54,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:54,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:54,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:54,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:54,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:54,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:54,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:54,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:54,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:54,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:54,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:54,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:54,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:54,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:54,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:54,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:55,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:55,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:55,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:55,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:55,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:55,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:55,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:55,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:55,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:55,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:55,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:55,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:55,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:55,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:55,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:55,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:55,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:55,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:55,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:55,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:55,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:55,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:55,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:55,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:55,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:55,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:56,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:56,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:56,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:56,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:56,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:56,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:56,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:56,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:56,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:56,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:56,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:56,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:56,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:56,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:56,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:56,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:57,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:57,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:57,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:57,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:57,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:57,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:57,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:57,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:57,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:57,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:57,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:57,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:57,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:57,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:57,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:57,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:57,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:57,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:57,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:57,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:57,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:57,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +0: [2023-02-09 22:49:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:57,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:58,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:58,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:58,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:58,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:58,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:58,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:58,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:58,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:58,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:58,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:58,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,401] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +0: [2023-02-09 22:49:58,404] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +7: [2023-02-09 22:49:58,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,413] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +7: [2023-02-09 22:49:58,417] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +5: [2023-02-09 22:49:58,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,418] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +5: [2023-02-09 22:49:58,422] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +4: [2023-02-09 22:49:58,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,422] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +2: [2023-02-09 22:49:58,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-09 22:49:58,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +4: [2023-02-09 22:49:58,426] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +2: [2023-02-09 22:49:58,428] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +2: [2023-02-09 22:49:58,429] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +7: [2023-02-09 22:49:58,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,440] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +6: [2023-02-09 22:49:58,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,442] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +7: [2023-02-09 22:49:58,444] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +6: [2023-02-09 22:49:58,446] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +3: [2023-02-09 22:49:58,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,447] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +1: [2023-02-09 22:49:58,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,448] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +3: [2023-02-09 22:49:58,451] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +1: [2023-02-09 22:49:58,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +5: [2023-02-09 22:49:58,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,457] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +4: [2023-02-09 22:49:58,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,459] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +5: [2023-02-09 22:49:58,461] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +4: [2023-02-09 22:49:58,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +4: [2023-02-09 22:49:58,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,473] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +4: [2023-02-09 22:49:58,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +4: [2023-02-09 22:49:58,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +6: [2023-02-09 22:49:58,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,476] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +4: [2023-02-09 22:49:58,477] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +4: [2023-02-09 22:49:58,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +4: [2023-02-09 22:49:58,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +6: [2023-02-09 22:49:58,480] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +0: [2023-02-09 22:49:58,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,484] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +7: [2023-02-09 22:49:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +3: [2023-02-09 22:49:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,488] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +5: [2023-02-09 22:49:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,488] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +5: [2023-02-09 22:49:58,488] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +7: [2023-02-09 22:49:58,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +3: [2023-02-09 22:49:58,492] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +5: [2023-02-09 22:49:58,492] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +1: [2023-02-09 22:49:58,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,497] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +1: [2023-02-09 22:49:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,499] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-09 22:49:58,501] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +4: [2023-02-09 22:49:58,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,502] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +1: [2023-02-09 22:49:58,503] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +3: [2023-02-09 22:49:58,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +5: [2023-02-09 22:49:58,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +4: [2023-02-09 22:49:58,506] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +2: [2023-02-09 22:49:58,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,508] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +3: [2023-02-09 22:49:58,508] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +5: [2023-02-09 22:49:58,509] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +5: [2023-02-09 22:49:58,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,510] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +2: [2023-02-09 22:49:58,512] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +5: [2023-02-09 22:49:58,514] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +7: [2023-02-09 22:49:58,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,515] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +6: [2023-02-09 22:49:58,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,517] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +4: [2023-02-09 22:49:58,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,517] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +3: [2023-02-09 22:49:58,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,519] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +2: [2023-02-09 22:49:58,519] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +6: [2023-02-09 22:49:58,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,520] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +6: [2023-02-09 22:49:58,520] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +6: [2023-02-09 22:49:58,520] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +4: [2023-02-09 22:49:58,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +3: [2023-02-09 22:49:58,523] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +6: [2023-02-09 22:49:58,524] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +2: [2023-02-09 22:49:58,524] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +0: [2023-02-09 22:49:58,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,527] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +3: [2023-02-09 22:49:58,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,527] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +7: [2023-02-09 22:49:58,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,530] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +0: [2023-02-09 22:49:58,531] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +3: [2023-02-09 22:49:58,531] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +7: [2023-02-09 22:49:58,534] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +2: [2023-02-09 22:49:58,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,535] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +1: [2023-02-09 22:49:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,537] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +3: [2023-02-09 22:49:58,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,538] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +2: [2023-02-09 22:49:58,538] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +1: [2023-02-09 22:49:58,541] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +6: [2023-02-09 22:49:58,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,541] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +3: [2023-02-09 22:49:58,542] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +3: [2023-02-09 22:49:58,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,543] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +6: [2023-02-09 22:49:58,546] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +3: [2023-02-09 22:49:58,547] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +4: [2023-02-09 22:49:58,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +1: [2023-02-09 22:49:58,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,550] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +1: [2023-02-09 22:49:58,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:58,553] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +1: [2023-02-09 22:49:58,552] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +1: [2023-02-09 22:49:58,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,553] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +5: [2023-02-09 22:49:58,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +5: [2023-02-09 22:49:58,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +1: [2023-02-09 22:49:58,557] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +1: [2023-02-09 22:49:58,557] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +5: [2023-02-09 22:49:58,558] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +5: [2023-02-09 22:49:58,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,564] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +5: [2023-02-09 22:49:58,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:58,564] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +6: [2023-02-09 22:49:58,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +5: [2023-02-09 22:49:58,568] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +6: [2023-02-09 22:49:58,568] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +5: [2023-02-09 22:49:58,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +2: [2023-02-09 22:49:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,576] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +1: [2023-02-09 22:49:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:58,580] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +2: [2023-02-09 22:49:58,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +1: [2023-02-09 22:49:58,584] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +7: [2023-02-09 22:49:58,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,585] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +7: [2023-02-09 22:49:58,590] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +6: [2023-02-09 22:49:58,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,591] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +7: [2023-02-09 22:49:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,592] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +2: [2023-02-09 22:49:58,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:58,593] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +0: [2023-02-09 22:49:58,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,594] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +6: [2023-02-09 22:49:58,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +7: [2023-02-09 22:49:58,596] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +2: [2023-02-09 22:49:58,597] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +0: [2023-02-09 22:49:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,598] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +0: [2023-02-09 22:49:58,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +0: [2023-02-09 22:49:58,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,600] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +0: [2023-02-09 22:49:58,605] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +0: [2023-02-09 22:49:58,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +3: [2023-02-09 22:49:58,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:58,625] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +7: [2023-02-09 22:49:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:58,627] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +3: [2023-02-09 22:49:58,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +7: [2023-02-09 22:49:58,632] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +6: [2023-02-09 22:49:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:58,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:58,656] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +0: [2023-02-09 22:49:58,656] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +6: [2023-02-09 22:49:58,660] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +0: [2023-02-09 22:49:58,660] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +0: [2023-02-09 22:49:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:59,704] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-09 22:49:59,712] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +2: [2023-02-09 22:49:59,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b71b5/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:59,733] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +2: [2023-02-09 22:49:59,737] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +0: successfully loaded checkpoint from checkpoints_619m2b71b5 at iteration 0 +7: time (ms) | load-checkpoint: 6567.94 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-09 22:49:59 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.031986 seconds +0: number of documents: 5641150 +0: > dataset split: +0: train: +0: document indices in [0, 5641150) total of 5641150 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.044 seconds +0: total number of samples: 1317244 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.035325 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.011 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-09 22:50:13 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 23940.95 | train/valid/test-data-iterators-setup: 13151.49 +0: [after training is done] datetime: 2023-02-09 22:50:13 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 3.409955E+00 | lm loss PPL: 3.026389E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 2820895: Thu 09 Feb 2023 10:50:45 PM EET diff --git a/619m2b71b5/sbatch_619m2b71b5.sh b/619m2b71b5/sbatch_619m2b71b5.sh new file mode 100755 index 0000000000000000000000000000000000000000..1e7fa86cc3768193ce2107e771cb04546c08dc7a --- /dev/null +++ b/619m2b71b5/sbatch_619m2b71b5.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b71b5 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1_308_594 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 13_086 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b71b5/sbatch_619m2b71b5val.sh b/619m2b71b5/sbatch_619m2b71b5val.sh new file mode 100644 index 0000000000000000000000000000000000000000..a0554801804a5651df8fd2529a7587c33261a43f --- /dev/null +++ b/619m2b71b5/sbatch_619m2b71b5val.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b71b5val +VARIANT_CKPT=619m2b71b5 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train2b7.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --no-load-optim \ + --reset-progress \ + --override-lr-scheduler \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b71b5/tensorboard_619m2b71b5/events.out.tfevents.1675878350.nid007115.114049.0 b/619m2b71b5/tensorboard_619m2b71b5/events.out.tfevents.1675878350.nid007115.114049.0 new file mode 100644 index 0000000000000000000000000000000000000000..095a3c3953090cfda98d60350845708be5a55cca --- /dev/null +++ b/619m2b71b5/tensorboard_619m2b71b5/events.out.tfevents.1675878350.nid007115.114049.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1c050a3a20036837bc24b2382bb47cacae5b808eacd7cb44eddba8ca8cf3882 +size 9099907 diff --git a/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675555841.nid005466.61856.0 b/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675555841.nid005466.61856.0 new file mode 100644 index 0000000000000000000000000000000000000000..ccc8f5c06d30e444cd2498038331fc3d97665c97 --- /dev/null +++ b/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675555841.nid005466.61856.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63cc7aa40db265e4ed7f274a26e7527dc7e5d5b19373cdb9182c618646782a3 +size 980 diff --git a/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675975736.nid005039.3607.0 b/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675975736.nid005039.3607.0 new file mode 100644 index 0000000000000000000000000000000000000000..479392e0f5a1733ee5194283df36b10ca0cd38b9 --- /dev/null +++ b/619m2b71b5/tensorboard_619m2b71b5val/events.out.tfevents.1675975736.nid005039.3607.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3cbafc4ddb741c3ae54b5d62fbb905876518c3f28e1bc0fdf5e6e162a3e3fac +size 980 diff --git a/619m2b71b5/transformers/config.json b/619m2b71b5/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a1592eaab83db706537cb6afd4da42ec9828de43 --- /dev/null +++ b/619m2b71b5/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1536, "n_layer": 19, "n_head": 12, "n_inner": 6144, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/619m2b71b5/transformers/pytorch_model.bin b/619m2b71b5/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..57316daac09e0097983247f49c7f965b79d4643f --- /dev/null +++ b/619m2b71b5/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391485844db9cd5608756e05b6cc01930f8f0d98bf1a2f22b1820b337e61c666 +size 1396896901 diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..17e2350233e122894291d79beb8ebb336754243a --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.24113227314208618, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.028811549749581045}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.055071873695941284, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011204612925514266}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2452044168577755, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004146181231017322}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08555555702198481, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016048215948057977}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.025762355208947534, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006991588869087533}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.12033245105308268, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002914084395965689}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04031658356103043, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010315530209771078}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.054089161495901594, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010851069651214552}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.24208069073651772, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004106759426399882}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.08413720771901391, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015616400503146003}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.053253809119784244, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010664007355544926}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.23782578142548957, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003955768184748774}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.08276144141135114, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015265102568023873}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..30174ad6f77ba6998c3ecb8475a6be30af7362aa --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.14254789969047976, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.019845474922838995}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.04821268198252875, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0010579107229228407}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.232679143877019, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0038247746748433682}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07483570357790037, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014590385445274368}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.020202864451372583, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006205320695817688}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.09387190930300064, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0024847529706070316}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03136893988102952, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009010096581650731}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.046699657881712986, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000997164628146769}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.22598685447558764, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036416397118735396}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07262116653857528, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013900825132693265}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.04645876164911166, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010110982905689784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.22267526889287845, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036045912797825898}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07210506647702201, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014053501287397759}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8e4f9565c11f6e795f807a27c3615f21f2ab87c8 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1308970942789927, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.016961383512188746}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.052649460677847454, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014219603058163019}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.21440506657491165, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003827575566693626}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07695919345861055, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015961219290256806}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02059879149840832, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007416229455782925}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.09007670334311271, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002519097222783427}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.031185481539925945, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009293186251700585}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05026535802475036, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013072494003329232}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.20834692375387437, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003703084178643004}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07398567839328438, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014846464327335155}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05039540248058442, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013436329978786829}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.20660337450142063, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036525158438921457}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0738855577509211, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015092163593500825}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..c2df3633a6d3230d9a8e5a14e988dfa19f1b2e9c --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.14159738625676038, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01328777336277722}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.0559017669330822, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001415504973773362}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.21020882835563734, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037778507945221773}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08020511457184787, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016658936756924728}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02161787414155675, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007378830973683681}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.08816181362986414, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002529493308982268}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.032205510029758244, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009805578541930325}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05293674301210271, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012614159260527502}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.20363111835546907, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036952382330166034}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0767227253342896, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015529863044902322}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05302531776083246, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012936166080435767}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.20183908009081986, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003615726755986769}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0765293941605234, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015601122827046713}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..24ae4afb252e995ce5188c4b1d1307ae5f738bc3 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.12749034254704042, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.008546817810079587}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05726731293902398, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015720916530496721}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2069366490025934, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0038164690952814466}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08088223963412505, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017198828655291519}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.021768832750612987, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007326243635228755}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.08853949896527306, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002553223455917552}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03241550862307877, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009843107469837493}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05420226613939085, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014232892445763313}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.20092063876168395, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003735513741233921}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07731419916529977, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015941439846477567}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05483296600876922, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014695599771991175}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.20051770757700416, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036882494796530423}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07778405525923675, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016222838654200084}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..91229a9a52e4b72e2736358b8802d8e69ebfba50 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.14672466430278214, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01564268820920237}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06292539518254431, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017764413833392624}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.21598479779779012, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003982796672051667}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08619882119239601, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017839516781522034}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02339631866881691, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008120233797503263}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.09353320709626459, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0026698138988629928}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03440062861070681, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010372568046984475}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05887445429613583, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001577669544709431}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.20886633456863324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003896797605122707}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.08167980777977522, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001621090928904429}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.059534473296970654, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016355653688013153}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.20802136342162436, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0037908739032240287}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.08205703397709067, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016450364163506247}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d08a7a7cd1f5d70a7ebc52e7b87144113b5111e9 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08120515498772168, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014566622323488952}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13089295969024178, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002138790758055214}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09183225246901366, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001446144986354779}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.007852382528498259, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003922058129387016}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.014832775763259401, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008057905909353507}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.00936006347003796, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00045703828961676785}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07192180042908321, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012150546841524432}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11788475031934322, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018866313289264249}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08185338476864322, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001222665407719195}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07602044340370688, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013542562970553641}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.12308023723290125, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020114849972917084}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.08606713776522688, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001341296792108946}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4746833238014895, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.033468841697744}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..831f4440cbbdd4096e0c3f7f4b0836f332297626 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.10417327087929776, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015293133070916412}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.16228858189077838, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022341452040226693}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.11664158636209752, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015303620143841674}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.010168495209728696, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004333723787968611}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.01767445100956555, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009100075263152787}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01172627441669315, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004907988758645163}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07914240123572493, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010437322612055133}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.12687740686127083, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017053545908882837}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08919330535243743, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010391926745202198}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.09843591549323863, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014274812071532774}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.15409355870617902, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021165601229461333}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.11031680935504037, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001426618711627084}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.6406128660470801, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.035859594691386595}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..79342940cd3671ffbfa04da48fb88a283bb53512 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.0963319401570562, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014967705867028737}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.1414450916345688, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020280581196838047}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.10401732048769743, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001405310541916511}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.007688727438441564, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00040590845811071865}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.012809419158605334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008177388044946671}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.008529492488429729, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004244131539762185}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07797302290028109, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011278685921172196}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11735550235323298, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00165972064507672}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08463427761436904, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010428084274352024}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.09051724366485044, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013894987389591678}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.13319160744817032, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018816103018270938}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09777333618790017, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0012982359349548577}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4930700792703116, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04205732595101384}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..5d04a7f5a299d598e64453aa9e0022be8e962bde --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08426872221089397, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017386132445443138}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.11184423591937163, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002071972670969457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.08452384126554126, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014785711167470451}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.007630966876501281, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004674881725363447}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.011142314286720276, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007599187546842887}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.007780455376255207, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00042969979080184644}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07064490274475228, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014062280303910137}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.0961295297113538, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001766017239716426}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.07135631553222517, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011811827490538645}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07886115447407943, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016407821239095097}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.10459876592461427, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019378349486849633}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.07884453658046324, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013692319014547523}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.5575488890409951, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04792209814991415}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..20162bd25afdfa637634449c6468576b84045c16 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.02970493332256511, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013488671512197252}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.038478094138158066, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0016371051576469045}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.02809270781704144, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001121638874100726}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.003199741133575067, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003382565178996562}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.004827494166756389, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006169335309909903}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0030248237722389558, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002667451855051004}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.025897853817423586, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011496396544348466}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.03414071229704594, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0014481845257237847}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.024641420864595455, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009641892588452016}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.027767807703340433, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012688231361680526}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.03600781213800578, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0015355693879018626}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.0261750250556484, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0010396781819747305}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.12506380809790343, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.020376015696282453}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..0d78b63636209584e767d5035419cd08d8d144db --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.0057152840599657825, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0006591061707793925}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.007158048021481606, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0008601289654793884}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.005155523162091939, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000539625563010443}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0006478442927818688, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0001481955341623784}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0013855253494764474, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004196160527606927}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0006805043621601407, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0001448105863097518}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.004940457010564996, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005670918341564018}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.006289222531320525, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007752251249677887}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.004472796916907503, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00046874640396691897}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.005405088855709713, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006342299220886388}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.006747668835613718, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0008229892431308825}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.004831960704635796, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005068548557139965}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.3355106654399238e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 4.1100081084908905e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8117aeb602a24c8cf36b73d8a55f48e15e1b0f --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.260189204618729, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04109337902247209}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.11971887919164963, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030700848126240347}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1617024874205441, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025644642437620013}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.11823687062241604, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019556352325943256}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.04054174307765198, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0019147408844450626}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.040364953765767445, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010209689659418585}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.03213244416920702, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009356250227933517}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.11289576504809533, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0027672672835420526}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.15853150815560624, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025376965513659185}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.11429846594242389, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001857455051103104}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.095108940518323, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002675969804009192}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.1230176876835313, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019806348045116966}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.09121706676453484, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015908648602271467}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bb7233365613961e899515e993df4c804b252dfc --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.682192729082199, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11115643858112895}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.32428153115227637, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002645445266730213}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.2742000744658761, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023208690537510268}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.2787366541178018, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002060360360450834}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.08120278348566318, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0016503581278321312}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.06856069612002233, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013223712649095532}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06983477129656213, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013083737751466344}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.23860589651916864, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002091926558809564}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2010399018853146, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017983085956935448}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.20375676150217123, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015670128786767876}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2684559413162685, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002392978011391298}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.22555334112360417, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020422052690831217}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.2296204517439229, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001846873774076903}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..5173088fa58f0a4d7483d6e3c45e5c37d115d38e --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.0370807194608984, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12956394178766437}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.2855909643330839, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003411790508354162}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.23383491841483692, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002863967372487143}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.24181974023597483, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0027155503537555077}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.07507496488912378, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00195553797287265}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.05960053641050613, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001343479649823018}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06138426309539808, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013147497691957174}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2104915776268624, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002734901496099888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.16941974652556596, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021320569811222414}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.17566514928441032, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020283646644480703}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.23804856738225783, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003026140977515158}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.19311658805746856, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002453387366018241}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.20004741465910422, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0023398756942231685}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1c61bb2c8268a06bb41ceb8804e822dbc5e22016 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.950513026047841, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1726499167889734}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.28523410237218244, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003745395287024571}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.22448967310033474, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029236877569921308}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.2342323601698636, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0028075156262294494}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.08226533229108834, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0024859106665703285}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.05875737556649167, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013832978171248258}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06106712127168939, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013548116894034825}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.21431650347706552, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00313026915062149}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.164369322711601, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002193355190986195}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.1721492556837031, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0021172930618799423}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2396892700674045, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0033835576983395218}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.18625647639365359, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025246775303922977}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.19463119037109028, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024298696568374094}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..429d05ed23597dd77d975aa8b60cf8c661809850 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.171878420319261, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13165410117653997}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3037844870072903, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003946062395029148}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.22615534444511, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028642415459267587}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.23829866168882227, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0027541705270047426}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.09595929727047421, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.003056744064807103}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.060334580497235495, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001371704738838684}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.06356480943581086, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013703160121874625}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.23221312530642865, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0034637855773476654}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.16662869932938643, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021647677396040422}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.17627726645390196, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020799317284272848}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.25797235414296243, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00365703607288398}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.18879057223376866, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025014527596644433}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.19915256616340277, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00239939399087058}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9b2cd0d1a326c8d405d2ff4d84911c48519086a2 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.487893054270047, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11030761825736048}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.331753242539869, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00391429042214275}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.24427841616686002, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002768263823097388}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.257756911063572, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002615487581334247}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.10947973402783245, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00326761197367824}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.06625511614929147, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001382292688307658}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.07032602009078999, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013827000891058656}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.25607308773270254, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00353770127140282}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.18152620055963956, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021065308567015063}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.1923849741767262, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001998001966591684}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2823049486328519, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003698492501181677}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.20403990476496298, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002418301091713723}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.21555168521441764, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022919133792166186}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_0.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..93de83faec6597c3343f4ec387701956b215c3df --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09543399279052138, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015401529274997991}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.23270186782334634, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0034796434136361463}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.13284331700228394, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002008733661615099}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.015354440981433308, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006868856257561475}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.04059213552758656, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018843299027419883}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.021952601017208886, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009814657106866197}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0844375279033619, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012633659097732559}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20731007297104162, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029542464284462223}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11780459518526487, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016583269901690515}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07242801558272945, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011726998663923092}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1789651394178854, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002834793999286031}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.10112098884045655, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015458255637941939}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.7652139574245598, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0580360191813792}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_1.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c6db51cd137c6bdf853cdfff2438615859afa36e --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08729205756601999, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014759119556582933}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2158560968459811, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033932985115285204}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12271943431877347, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001993408251820057}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011295310908611753, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000637529563442929}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.028485623386367155, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015843238636349178}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015987682957254674, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008920064252614353}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07627791797846474, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012224414385125447}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18997477453945114, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002884323973027333}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10744881438263615, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016582533109032562}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06792038786639965, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011490798773779924}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1699002900122474, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002766652887778396}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09577867803568567, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015679685291023308}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6234335017256344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10516892162348979}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_2.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..5526f9a9720b5ff2c0cf0fbb1bb8607c8c209a41 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08140627832123237, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013538162390964604}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.20395783512684038, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003180562989448791}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11490970288671254, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018335229977129786}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.009805619707603953, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005371390414333858}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02562908685582355, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013923986683927981}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.014011395875766086, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007593569146491094}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07319916634188256, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011294377025329568}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18460546669295397, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00275312894855031}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10350576354209164, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015393600143767505}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.062398343344353395, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010008295514121795}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.15838240344246676, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00249154359545472}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.08839514462271125, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013762224046845657}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.47969693901774124, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10393002306967618}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_3.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..557c02c801e9260e1c98e7c0ffe81ccdd702010b --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0826381581304149, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016168276430261522}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.1976603680004612, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003302591417675845}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11336019965791448, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019485384501665736}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.010818142362260436, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005875954184048923}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0270483375276355, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014259889511806186}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015080688670459261, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007967006531578405}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0739135150808346, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013246344100743559}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.1789528343212182, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0028966911378406277}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10193945159447333, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001655352593744887}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06482067265504902, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012271522821360112}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1566489179441027, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026261509927438605}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.08914872966484293, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014930581583732095}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5334095301261237, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05904694475713066}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_4.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ac1c968fa257e6a5dc2d776529f18b4bb75289c5 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.029268596464183467, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020259399835436914}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.05199391413839539, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030746410400925525}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.03339951614864071, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00198021955667841}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.004012419731989478, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006226238446139109}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.007824394253170313, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009120531748140449}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0048903937329160254, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006123327257305959}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.025666783203699015, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017988634183294424}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04584486704573437, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027083142064180553}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.029149042347803685, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016983449338329505}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.023673801967461685, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017485680903315058}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.04082810902018552, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00242874515124222}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.026316949881964886, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015704230017045268}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.34520892365141886, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05969467959070213}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_5.json b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..e81cbb86fc6be8c49b8dc9d689a86a6571009a88 --- /dev/null +++ b/619m2b72b7/evaluation/generation/agg.619m2b72b7_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0028853675321050755, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.000821656688738579}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.002467639720675758, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.000687296409234268}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.002577821388984159, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007153291957254458}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0006183705614893184, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0002545551974709719}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0005564448483865506, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0002613253273684741}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0005725459337607456, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00025123537297203486}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0023823600638835174, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006598065019191697}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.002095069624026878, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005904857317724273}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.002155387904801103, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005925129628611262}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0024395355927571594, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006863660036715112}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.002124643173444279, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.000598494012486739}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0021943712199422225, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006063473391893399}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.0285852034656401e-39, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.7605427719028158e-33}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..027762755ee57dc33fe5f801f0b9e484666cc225 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c5d232e82c76ca9dce6e4d94aa6d33e197fce673b8375b995030d2072fbf8f1 +size 4151745 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..005b88323a7c09eaa83d6697667bd191157b6838 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e2fa86422d03761d88981a00e8dbf2666f4658c32eac446ca1b12bcbea7482 +size 5201369 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fad4886e64795f7d2ac681e40df5e4617a6d6fb2 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06912efa37140001c964472fcc16d3d94aadf558ff133863e3ee180e1e953706 +size 5991438 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..94b9995abe28bbfbef9a37e77687b4759245b339 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2bce74f73147e9c24afa9ed1ad32afe090c226f6891ef04165bff94d2cf564a +size 6839170 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5ae874d71cffaaa80f8912684bb5f2b322ac920b --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0c98ee152a96ae53b25f9041f0ee2c6179a7d901012b8d93aca5d275cb9f35c +size 7709195 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ab3a9135f099566f2b3b431623bcb0bcbef92a1b --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e14697d4f48c8ea97d291e5a63e5412b2ec4a20abdb33f183d349894336b78d2 +size 8571033 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..036400a01def1549e0711dff50a6d372ed0fd046 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ae0e15f4ccf3799f5732ddfc1fd3668e6ec55c2a0d9867ca049ce2d9882a18c +size 7605924 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6930e94f6803f040e2cb215a84c9b861ee1864bc --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030e94d5c462356d895b0551efbad34e0da29681d05058322e00d7d25b8c825c +size 13325131 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5343085edb32cabfaaa323af1881355aa1feec01 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a6f2856bcb49f0a17a2561b6bc1e4d055aebffdd9d1a027b101d4bb5cbb4fd +size 18902290 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..59e488d616dce8cc26612141ce73edf705367e33 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef7ae46ec216bd45778f3255a69710d2f5ede401127a5377973cca0cea43f03f +size 24305507 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8ab8fe4ed83b4ffb396a6017cc15b3de8f97801e --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c4ef4f11784cf57bd3ecf32500e2dddf96c6d5816691a0ac1ab2e55cb7404a +size 29461828 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..69dad2b27db28de1d9b7be390e19ad11a69812e0 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14cb987f8542723acb6bc3da05a3ed0d97c7d148829fc540488484fc993b9c46 +size 34796665 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e49c1c3533d641ddf8f25066d81e488e2d50333c --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64187cdbd08c5cb4cb3e984e3f6d7d78b9a7986e8cfc5e7c64ddeeb5e3397ecc +size 4100405 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..61924cd0c535931fe1c381a8a75626f421468dfe --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16a82ed4a4d29fd81ab95b0f6e222b65764ce40140455d6ddc4e41d90a75b914 +size 5062724 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1bbebf13c2246c6e1e0d77fb9c27475316047b40 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fbb8dfa93fceaedb9ed626e2bfc5fb4e6335242d8085cf78690c9fd96f7aeb3 +size 6063993 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cd6e1e29d898857e717dc7d591f99d702f6b5cdb --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86aa6526bbb6cff7331c6a8c877fb2b32e33bc7473338a884f99a0ed4aa4f479 +size 7128503 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7faf43e9a1e48455cad9442b532a319875ea2328 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:745ccb82977e03afd996cf639b537c87b0ee73a17ee90ea531108f9aaee7bb06 +size 8209516 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c2ae1db514ded49feec17925c481f912be3a757e --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:282872154bad836c2551bd98598c0e9d99b6d0135384ab75b05d5c5451e0aa8d +size 9313165 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_0.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5c2524b039c46ee2986b90984847e1e451313980 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f667173c96aeeb3e1ee121692b843655c7f8aab905d428341f25895a8f18256f +size 2833884 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_1.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..666c6ec0bd282360d4e53f61f3fbbc8f15c55e9f --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff12fd65dbd099ee97f78dc8cf8896fea5a6ee6975d701a9ae25364012f4296 +size 5108804 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_2.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..90e87f7df5e914c62f2e430a784401865388dd16 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0ccdfbe664710c16c9151abd43085aa8e5d37782531aa25ab84f5c3a187f87 +size 7385078 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_3.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bb4729e7b9f1b3fe58fbce54c0d77d4a5235ff34 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:351e2242bdab94e0747f708bcdc7b65b4d0a58eda3fe96fd2d598f36b855724a +size 9652066 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_4.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6cf093aea94b3bbb2522f3285af095b391886dc3 --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb57a17f6cba2bc2f492487b561db68a97bb2ef3b415f77302227a1039f7f66 +size 11674464 diff --git a/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_5.jsonl b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0f448cf8ba2e32c9744f375da2c0b49b691b3e1e --- /dev/null +++ b/619m2b72b7/evaluation/generation/examples.619m2b72b7_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85171f9cd529d2ed069a0b229d15552875a0e087c7593dcaec36bdf798e98030 +size 13897540 diff --git a/619m2b72b7/evaluation/generation/merged.csv b/619m2b72b7/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..6cbadbe0699d63ef6ce1180673970fa1325dde3a --- /dev/null +++ b/619m2b72b7/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.03213244416920702 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.03213244416920702 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.06983477129656213 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.06983477129656213 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.06138426309539808 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.06138426309539808 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.06106712127168939 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.06106712127168939 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.06356480943581086 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.06356480943581086 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.07032602009078999 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.07032602009078999 +e2e_nlg_cleaned,5,average,multiple,0.059718238226576244 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.021952601017208886 +gem_xsum,0,median,rouge2_fmeasure,0.021952601017208886 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.015987682957254674 +gem_xsum,1,median,rouge2_fmeasure,0.015987682957254674 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.014011395875766086 +gem_xsum,2,median,rouge2_fmeasure,0.014011395875766086 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.015080688670459261 +gem_xsum,3,median,rouge2_fmeasure,0.015080688670459261 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.0048903937329160254 +gem_xsum,4,median,rouge2_fmeasure,0.0048903937329160254 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0005725459337607456 +gem_xsum,5,median,rouge2_fmeasure,0.0005725459337607456 +gem_xsum,5,average,multiple,0.012082551364560946 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.04031658356103043 +web_nlg_en,0,median,rouge2_fmeasure,0.04031658356103043 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.03136893988102952 +web_nlg_en,1,median,rouge2_fmeasure,0.03136893988102952 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.031185481539925945 +web_nlg_en,2,median,rouge2_fmeasure,0.031185481539925945 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.032205510029758244 +web_nlg_en,3,median,rouge2_fmeasure,0.032205510029758244 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.03241550862307877 +web_nlg_en,4,median,rouge2_fmeasure,0.03241550862307877 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.03440062861070681 +web_nlg_en,5,median,rouge2_fmeasure,0.03440062861070681 +web_nlg_en,5,average,multiple,0.03364877537425495 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.00936006347003796 +wiki_lingua_en,0,median,rouge2_fmeasure,0.00936006347003796 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.01172627441669315 +wiki_lingua_en,1,median,rouge2_fmeasure,0.01172627441669315 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.008529492488429729 +wiki_lingua_en,2,median,rouge2_fmeasure,0.008529492488429729 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.007780455376255207 +wiki_lingua_en,3,median,rouge2_fmeasure,0.007780455376255207 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.0030248237722389558 +wiki_lingua_en,4,median,rouge2_fmeasure,0.0030248237722389558 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.0006805043621601407 +wiki_lingua_en,5,median,rouge2_fmeasure,0.0006805043621601407 +wiki_lingua_en,5,average,multiple,0.00685026898096919 diff --git a/619m2b72b7/evaluation/generation/merged.json b/619m2b72b7/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..1e68b1344af496618deb1bdd912ba3d4d530d21e --- /dev/null +++ b/619m2b72b7/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.24113227314208618, "bleu_stderr": 0.028811549749581045, "rouge1_fmeasure": 0.08555555702198481, "rouge1_fmeasure_stderr": 0.0016048215948057977, "rouge1_precision": 0.055071873695941284, "rouge1_precision_stderr": 0.0011204612925514266, "rouge1_recall": 0.2452044168577755, "rouge1_recall_stderr": 0.004146181231017322, "rouge2_fmeasure": 0.04031658356103043, "rouge2_fmeasure_stderr": 0.0010315530209771078, "rouge2_precision": 0.025762355208947534, "rouge2_precision_stderr": 0.0006991588869087533, "rouge2_recall": 0.12033245105308268, "rouge2_recall_stderr": 0.002914084395965689, "rougeL_fmeasure": 0.08413720771901391, "rougeL_fmeasure_stderr": 0.0015616400503146003, "rougeL_precision": 0.054089161495901594, "rougeL_precision_stderr": 0.0010851069651214552, "rougeL_recall": 0.24208069073651772, "rougeL_recall_stderr": 0.004106759426399882, "rougeLsum_fmeasure": 0.08276144141135114, "rougeLsum_fmeasure_stderr": 0.0015265102568023873, "rougeLsum_precision": 0.053253809119784244, "rougeLsum_precision_stderr": 0.0010664007355544926, "rougeLsum_recall": 0.23782578142548957, "rougeLsum_recall_stderr": 0.003955768184748774}}, "1": {"PALM_prompt": {"bleu": 0.14254789969047976, "bleu_stderr": 0.019845474922838995, "rouge1_fmeasure": 0.07483570357790037, "rouge1_fmeasure_stderr": 0.0014590385445274368, "rouge1_precision": 0.04821268198252875, "rouge1_precision_stderr": 0.0010579107229228407, "rouge1_recall": 0.232679143877019, "rouge1_recall_stderr": 0.0038247746748433682, "rouge2_fmeasure": 0.03136893988102952, "rouge2_fmeasure_stderr": 0.0009010096581650731, "rouge2_precision": 0.020202864451372583, "rouge2_precision_stderr": 0.0006205320695817688, "rouge2_recall": 0.09387190930300064, "rouge2_recall_stderr": 0.0024847529706070316, "rougeL_fmeasure": 0.07262116653857528, "rougeL_fmeasure_stderr": 0.0013900825132693265, "rougeL_precision": 0.046699657881712986, "rougeL_precision_stderr": 0.000997164628146769, "rougeL_recall": 0.22598685447558764, "rougeL_recall_stderr": 0.0036416397118735396, "rougeLsum_fmeasure": 0.07210506647702201, "rougeLsum_fmeasure_stderr": 0.0014053501287397759, "rougeLsum_precision": 0.04645876164911166, "rougeLsum_precision_stderr": 0.0010110982905689784, "rougeLsum_recall": 0.22267526889287845, "rougeLsum_recall_stderr": 0.0036045912797825898}}, "2": {"PALM_prompt": {"bleu": 0.1308970942789927, "bleu_stderr": 0.016961383512188746, "rouge1_fmeasure": 0.07695919345861055, "rouge1_fmeasure_stderr": 0.0015961219290256806, "rouge1_precision": 0.052649460677847454, "rouge1_precision_stderr": 0.0014219603058163019, "rouge1_recall": 0.21440506657491165, "rouge1_recall_stderr": 0.003827575566693626, "rouge2_fmeasure": 0.031185481539925945, "rouge2_fmeasure_stderr": 0.0009293186251700585, "rouge2_precision": 0.02059879149840832, "rouge2_precision_stderr": 0.0007416229455782925, "rouge2_recall": 0.09007670334311271, "rouge2_recall_stderr": 0.002519097222783427, "rougeL_fmeasure": 0.07398567839328438, "rougeL_fmeasure_stderr": 0.0014846464327335155, "rougeL_precision": 0.05026535802475036, "rougeL_precision_stderr": 0.0013072494003329232, "rougeL_recall": 0.20834692375387437, "rougeL_recall_stderr": 0.003703084178643004, "rougeLsum_fmeasure": 0.0738855577509211, "rougeLsum_fmeasure_stderr": 0.0015092163593500825, "rougeLsum_precision": 0.05039540248058442, "rougeLsum_precision_stderr": 0.0013436329978786829, "rougeLsum_recall": 0.20660337450142063, "rougeLsum_recall_stderr": 0.0036525158438921457}}, "3": {"PALM_prompt": {"bleu": 0.14159738625676038, "bleu_stderr": 0.01328777336277722, "rouge1_fmeasure": 0.08020511457184787, "rouge1_fmeasure_stderr": 0.0016658936756924728, "rouge1_precision": 0.0559017669330822, "rouge1_precision_stderr": 0.001415504973773362, "rouge1_recall": 0.21020882835563734, "rouge1_recall_stderr": 0.0037778507945221773, "rouge2_fmeasure": 0.032205510029758244, "rouge2_fmeasure_stderr": 0.0009805578541930325, "rouge2_precision": 0.02161787414155675, "rouge2_precision_stderr": 0.0007378830973683681, "rouge2_recall": 0.08816181362986414, "rouge2_recall_stderr": 0.002529493308982268, "rougeL_fmeasure": 0.0767227253342896, "rougeL_fmeasure_stderr": 0.0015529863044902322, "rougeL_precision": 0.05293674301210271, "rougeL_precision_stderr": 0.0012614159260527502, "rougeL_recall": 0.20363111835546907, "rougeL_recall_stderr": 0.0036952382330166034, "rougeLsum_fmeasure": 0.0765293941605234, "rougeLsum_fmeasure_stderr": 0.0015601122827046713, "rougeLsum_precision": 0.05302531776083246, "rougeLsum_precision_stderr": 0.0012936166080435767, "rougeLsum_recall": 0.20183908009081986, "rougeLsum_recall_stderr": 0.003615726755986769}}, "4": {"PALM_prompt": {"bleu": 0.12749034254704042, "bleu_stderr": 0.008546817810079587, "rouge1_fmeasure": 0.08088223963412505, "rouge1_fmeasure_stderr": 0.0017198828655291519, "rouge1_precision": 0.05726731293902398, "rouge1_precision_stderr": 0.0015720916530496721, "rouge1_recall": 0.2069366490025934, "rouge1_recall_stderr": 0.0038164690952814466, "rouge2_fmeasure": 0.03241550862307877, "rouge2_fmeasure_stderr": 0.0009843107469837493, "rouge2_precision": 0.021768832750612987, "rouge2_precision_stderr": 0.0007326243635228755, "rouge2_recall": 0.08853949896527306, "rouge2_recall_stderr": 0.002553223455917552, "rougeL_fmeasure": 0.07731419916529977, "rougeL_fmeasure_stderr": 0.0015941439846477567, "rougeL_precision": 0.05420226613939085, "rougeL_precision_stderr": 0.0014232892445763313, "rougeL_recall": 0.20092063876168395, "rougeL_recall_stderr": 0.003735513741233921, "rougeLsum_fmeasure": 0.07778405525923675, "rougeLsum_fmeasure_stderr": 0.0016222838654200084, "rougeLsum_precision": 0.05483296600876922, "rougeLsum_precision_stderr": 0.0014695599771991175, "rougeLsum_recall": 0.20051770757700416, "rougeLsum_recall_stderr": 0.0036882494796530423}}, "5": {"PALM_prompt": {"bleu": 0.14672466430278214, "bleu_stderr": 0.01564268820920237, "rouge1_fmeasure": 0.08619882119239601, "rouge1_fmeasure_stderr": 0.0017839516781522034, "rouge1_precision": 0.06292539518254431, "rouge1_precision_stderr": 0.0017764413833392624, "rouge1_recall": 0.21598479779779012, "rouge1_recall_stderr": 0.003982796672051667, "rouge2_fmeasure": 0.03440062861070681, "rouge2_fmeasure_stderr": 0.0010372568046984475, "rouge2_precision": 0.02339631866881691, "rouge2_precision_stderr": 0.0008120233797503263, "rouge2_recall": 0.09353320709626459, "rouge2_recall_stderr": 0.0026698138988629928, "rougeL_fmeasure": 0.08167980777977522, "rougeL_fmeasure_stderr": 0.001621090928904429, "rougeL_precision": 0.05887445429613583, "rougeL_precision_stderr": 0.001577669544709431, "rougeL_recall": 0.20886633456863324, "rougeL_recall_stderr": 0.003896797605122707, "rougeLsum_fmeasure": 0.08205703397709067, "rougeLsum_fmeasure_stderr": 0.0016450364163506247, "rougeLsum_precision": 0.059534473296970654, "rougeLsum_precision_stderr": 0.0016355653688013153, "rougeLsum_recall": 0.20802136342162436, "rougeLsum_recall_stderr": 0.0037908739032240287}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 0.4746833238014895, "bleu_stderr": 0.033468841697744, "rouge1_fmeasure": 0.09183225246901366, "rouge1_fmeasure_stderr": 0.001446144986354779, "rouge1_precision": 0.08120515498772168, "rouge1_precision_stderr": 0.0014566622323488952, "rouge1_recall": 0.13089295969024178, "rouge1_recall_stderr": 0.002138790758055214, "rouge2_fmeasure": 0.00936006347003796, "rouge2_fmeasure_stderr": 0.00045703828961676785, "rouge2_precision": 0.007852382528498259, "rouge2_precision_stderr": 0.0003922058129387016, "rouge2_recall": 0.014832775763259401, "rouge2_recall_stderr": 0.0008057905909353507, "rougeL_fmeasure": 0.08185338476864322, "rougeL_fmeasure_stderr": 0.001222665407719195, "rougeL_precision": 0.07192180042908321, "rougeL_precision_stderr": 0.0012150546841524432, "rougeL_recall": 0.11788475031934322, "rougeL_recall_stderr": 0.0018866313289264249, "rougeLsum_fmeasure": 0.08606713776522688, "rougeLsum_fmeasure_stderr": 0.001341296792108946, "rougeLsum_precision": 0.07602044340370688, "rougeLsum_precision_stderr": 0.0013542562970553641, "rougeLsum_recall": 0.12308023723290125, "rougeLsum_recall_stderr": 0.0020114849972917084}}, "1": {"tldr_en": {"bleu": 0.6406128660470801, "bleu_stderr": 0.035859594691386595, "rouge1_fmeasure": 0.11664158636209752, "rouge1_fmeasure_stderr": 0.0015303620143841674, "rouge1_precision": 0.10417327087929776, "rouge1_precision_stderr": 0.0015293133070916412, "rouge1_recall": 0.16228858189077838, "rouge1_recall_stderr": 0.0022341452040226693, "rouge2_fmeasure": 0.01172627441669315, "rouge2_fmeasure_stderr": 0.0004907988758645163, "rouge2_precision": 0.010168495209728696, "rouge2_precision_stderr": 0.0004333723787968611, "rouge2_recall": 0.01767445100956555, "rouge2_recall_stderr": 0.0009100075263152787, "rougeL_fmeasure": 0.08919330535243743, "rougeL_fmeasure_stderr": 0.0010391926745202198, "rougeL_precision": 0.07914240123572493, "rougeL_precision_stderr": 0.0010437322612055133, "rougeL_recall": 0.12687740686127083, "rougeL_recall_stderr": 0.0017053545908882837, "rougeLsum_fmeasure": 0.11031680935504037, "rougeLsum_fmeasure_stderr": 0.001426618711627084, "rougeLsum_precision": 0.09843591549323863, "rougeLsum_precision_stderr": 0.0014274812071532774, "rougeLsum_recall": 0.15409355870617902, "rougeLsum_recall_stderr": 0.0021165601229461333}}, "2": {"tldr_en": {"bleu": 0.4930700792703116, "bleu_stderr": 0.04205732595101384, "rouge1_fmeasure": 0.10401732048769743, "rouge1_fmeasure_stderr": 0.001405310541916511, "rouge1_precision": 0.0963319401570562, "rouge1_precision_stderr": 0.0014967705867028737, "rouge1_recall": 0.1414450916345688, "rouge1_recall_stderr": 0.0020280581196838047, "rouge2_fmeasure": 0.008529492488429729, "rouge2_fmeasure_stderr": 0.0004244131539762185, "rouge2_precision": 0.007688727438441564, "rouge2_precision_stderr": 0.00040590845811071865, "rouge2_recall": 0.012809419158605334, "rouge2_recall_stderr": 0.0008177388044946671, "rougeL_fmeasure": 0.08463427761436904, "rougeL_fmeasure_stderr": 0.0010428084274352024, "rougeL_precision": 0.07797302290028109, "rougeL_precision_stderr": 0.0011278685921172196, "rougeL_recall": 0.11735550235323298, "rougeL_recall_stderr": 0.00165972064507672, "rougeLsum_fmeasure": 0.09777333618790017, "rougeLsum_fmeasure_stderr": 0.0012982359349548577, "rougeLsum_precision": 0.09051724366485044, "rougeLsum_precision_stderr": 0.0013894987389591678, "rougeLsum_recall": 0.13319160744817032, "rougeLsum_recall_stderr": 0.0018816103018270938}}, "3": {"tldr_en": {"bleu": 0.5575488890409951, "bleu_stderr": 0.04792209814991415, "rouge1_fmeasure": 0.08452384126554126, "rouge1_fmeasure_stderr": 0.0014785711167470451, "rouge1_precision": 0.08426872221089397, "rouge1_precision_stderr": 0.0017386132445443138, "rouge1_recall": 0.11184423591937163, "rouge1_recall_stderr": 0.002071972670969457, "rouge2_fmeasure": 0.007780455376255207, "rouge2_fmeasure_stderr": 0.00042969979080184644, "rouge2_precision": 0.007630966876501281, "rouge2_precision_stderr": 0.0004674881725363447, "rouge2_recall": 0.011142314286720276, "rouge2_recall_stderr": 0.0007599187546842887, "rougeL_fmeasure": 0.07135631553222517, "rougeL_fmeasure_stderr": 0.0011811827490538645, "rougeL_precision": 0.07064490274475228, "rougeL_precision_stderr": 0.0014062280303910137, "rougeL_recall": 0.0961295297113538, "rougeL_recall_stderr": 0.001766017239716426, "rougeLsum_fmeasure": 0.07884453658046324, "rougeLsum_fmeasure_stderr": 0.0013692319014547523, "rougeLsum_precision": 0.07886115447407943, "rougeLsum_precision_stderr": 0.0016407821239095097, "rougeLsum_recall": 0.10459876592461427, "rougeLsum_recall_stderr": 0.0019378349486849633}}, "4": {"tldr_en": {"bleu": 0.12506380809790343, "bleu_stderr": 0.020376015696282453, "rouge1_fmeasure": 0.02809270781704144, "rouge1_fmeasure_stderr": 0.001121638874100726, "rouge1_precision": 0.02970493332256511, "rouge1_precision_stderr": 0.0013488671512197252, "rouge1_recall": 0.038478094138158066, "rouge1_recall_stderr": 0.0016371051576469045, "rouge2_fmeasure": 0.0030248237722389558, "rouge2_fmeasure_stderr": 0.0002667451855051004, "rouge2_precision": 0.003199741133575067, "rouge2_precision_stderr": 0.0003382565178996562, "rouge2_recall": 0.004827494166756389, "rouge2_recall_stderr": 0.0006169335309909903, "rougeL_fmeasure": 0.024641420864595455, "rougeL_fmeasure_stderr": 0.0009641892588452016, "rougeL_precision": 0.025897853817423586, "rougeL_precision_stderr": 0.0011496396544348466, "rougeL_recall": 0.03414071229704594, "rougeL_recall_stderr": 0.0014481845257237847, "rougeLsum_fmeasure": 0.0261750250556484, "rougeLsum_fmeasure_stderr": 0.0010396781819747305, "rougeLsum_precision": 0.027767807703340433, "rougeLsum_precision_stderr": 0.0012688231361680526, "rougeLsum_recall": 0.03600781213800578, "rougeLsum_recall_stderr": 0.0015355693879018626}}, "5": {"tldr_en": {"bleu": 1.3355106654399238e-07, "bleu_stderr": 4.1100081084908905e-07, "rouge1_fmeasure": 0.005155523162091939, "rouge1_fmeasure_stderr": 0.000539625563010443, "rouge1_precision": 0.0057152840599657825, "rouge1_precision_stderr": 0.0006591061707793925, "rouge1_recall": 0.007158048021481606, "rouge1_recall_stderr": 0.0008601289654793884, "rouge2_fmeasure": 0.0006805043621601407, "rouge2_fmeasure_stderr": 0.0001448105863097518, "rouge2_precision": 0.0006478442927818688, "rouge2_precision_stderr": 0.0001481955341623784, "rouge2_recall": 0.0013855253494764474, "rouge2_recall_stderr": 0.0004196160527606927, "rougeL_fmeasure": 0.004472796916907503, "rougeL_fmeasure_stderr": 0.00046874640396691897, "rougeL_precision": 0.004940457010564996, "rougeL_precision_stderr": 0.0005670918341564018, "rougeL_recall": 0.006289222531320525, "rougeL_recall_stderr": 0.0007752251249677887, "rougeLsum_fmeasure": 0.004831960704635796, "rougeLsum_fmeasure_stderr": 0.0005068548557139965, "rougeLsum_precision": 0.005405088855709713, "rougeLsum_precision_stderr": 0.0006342299220886388, "rougeLsum_recall": 0.006747668835613718, "rougeLsum_recall_stderr": 0.0008229892431308825}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 2.260189204618729, "bleu_stderr": 0.04109337902247209, "rouge1_fmeasure": 0.11823687062241604, "rouge1_fmeasure_stderr": 0.0019556352325943256, "rouge1_precision": 0.11971887919164963, "rouge1_precision_stderr": 0.0030700848126240347, "rouge1_recall": 0.1617024874205441, "rouge1_recall_stderr": 0.0025644642437620013, "rouge2_fmeasure": 0.03213244416920702, "rouge2_fmeasure_stderr": 0.0009356250227933517, "rouge2_precision": 0.04054174307765198, "rouge2_precision_stderr": 0.0019147408844450626, "rouge2_recall": 0.040364953765767445, "rouge2_recall_stderr": 0.0010209689659418585, "rougeL_fmeasure": 0.11429846594242389, "rougeL_fmeasure_stderr": 0.001857455051103104, "rougeL_precision": 0.11289576504809533, "rougeL_precision_stderr": 0.0027672672835420526, "rougeL_recall": 0.15853150815560624, "rougeL_recall_stderr": 0.0025376965513659185, "rougeLsum_fmeasure": 0.09121706676453484, "rougeLsum_fmeasure_stderr": 0.0015908648602271467, "rougeLsum_precision": 0.095108940518323, "rougeLsum_precision_stderr": 0.002675969804009192, "rougeLsum_recall": 0.1230176876835313, "rougeLsum_recall_stderr": 0.0019806348045116966}}, "1": {"generate_text_restaurant": {"bleu": 3.682192729082199, "bleu_stderr": 0.11115643858112895, "rouge1_fmeasure": 0.2787366541178018, "rouge1_fmeasure_stderr": 0.002060360360450834, "rouge1_precision": 0.32428153115227637, "rouge1_precision_stderr": 0.002645445266730213, "rouge1_recall": 0.2742000744658761, "rouge1_recall_stderr": 0.0023208690537510268, "rouge2_fmeasure": 0.06983477129656213, "rouge2_fmeasure_stderr": 0.0013083737751466344, "rouge2_precision": 0.08120278348566318, "rouge2_precision_stderr": 0.0016503581278321312, "rouge2_recall": 0.06856069612002233, "rouge2_recall_stderr": 0.0013223712649095532, "rougeL_fmeasure": 0.20375676150217123, "rougeL_fmeasure_stderr": 0.0015670128786767876, "rougeL_precision": 0.23860589651916864, "rougeL_precision_stderr": 0.002091926558809564, "rougeL_recall": 0.2010399018853146, "rougeL_recall_stderr": 0.0017983085956935448, "rougeLsum_fmeasure": 0.2296204517439229, "rougeLsum_fmeasure_stderr": 0.001846873774076903, "rougeLsum_precision": 0.2684559413162685, "rougeLsum_precision_stderr": 0.002392978011391298, "rougeLsum_recall": 0.22555334112360417, "rougeLsum_recall_stderr": 0.0020422052690831217}}, "2": {"generate_text_restaurant": {"bleu": 3.0370807194608984, "bleu_stderr": 0.12956394178766437, "rouge1_fmeasure": 0.24181974023597483, "rouge1_fmeasure_stderr": 0.0027155503537555077, "rouge1_precision": 0.2855909643330839, "rouge1_precision_stderr": 0.003411790508354162, "rouge1_recall": 0.23383491841483692, "rouge1_recall_stderr": 0.002863967372487143, "rouge2_fmeasure": 0.06138426309539808, "rouge2_fmeasure_stderr": 0.0013147497691957174, "rouge2_precision": 0.07507496488912378, "rouge2_precision_stderr": 0.00195553797287265, "rouge2_recall": 0.05960053641050613, "rouge2_recall_stderr": 0.001343479649823018, "rougeL_fmeasure": 0.17566514928441032, "rougeL_fmeasure_stderr": 0.0020283646644480703, "rougeL_precision": 0.2104915776268624, "rougeL_precision_stderr": 0.002734901496099888, "rougeL_recall": 0.16941974652556596, "rougeL_recall_stderr": 0.0021320569811222414, "rougeLsum_fmeasure": 0.20004741465910422, "rougeLsum_fmeasure_stderr": 0.0023398756942231685, "rougeLsum_precision": 0.23804856738225783, "rougeLsum_precision_stderr": 0.003026140977515158, "rougeLsum_recall": 0.19311658805746856, "rougeLsum_recall_stderr": 0.002453387366018241}}, "3": {"generate_text_restaurant": {"bleu": 2.950513026047841, "bleu_stderr": 0.1726499167889734, "rouge1_fmeasure": 0.2342323601698636, "rouge1_fmeasure_stderr": 0.0028075156262294494, "rouge1_precision": 0.28523410237218244, "rouge1_precision_stderr": 0.003745395287024571, "rouge1_recall": 0.22448967310033474, "rouge1_recall_stderr": 0.0029236877569921308, "rouge2_fmeasure": 0.06106712127168939, "rouge2_fmeasure_stderr": 0.0013548116894034825, "rouge2_precision": 0.08226533229108834, "rouge2_precision_stderr": 0.0024859106665703285, "rouge2_recall": 0.05875737556649167, "rouge2_recall_stderr": 0.0013832978171248258, "rougeL_fmeasure": 0.1721492556837031, "rougeL_fmeasure_stderr": 0.0021172930618799423, "rougeL_precision": 0.21431650347706552, "rougeL_precision_stderr": 0.00313026915062149, "rougeL_recall": 0.164369322711601, "rougeL_recall_stderr": 0.002193355190986195, "rougeLsum_fmeasure": 0.19463119037109028, "rougeLsum_fmeasure_stderr": 0.0024298696568374094, "rougeLsum_precision": 0.2396892700674045, "rougeLsum_precision_stderr": 0.0033835576983395218, "rougeLsum_recall": 0.18625647639365359, "rougeLsum_recall_stderr": 0.0025246775303922977}}, "4": {"generate_text_restaurant": {"bleu": 3.171878420319261, "bleu_stderr": 0.13165410117653997, "rouge1_fmeasure": 0.23829866168882227, "rouge1_fmeasure_stderr": 0.0027541705270047426, "rouge1_precision": 0.3037844870072903, "rouge1_precision_stderr": 0.003946062395029148, "rouge1_recall": 0.22615534444511, "rouge1_recall_stderr": 0.0028642415459267587, "rouge2_fmeasure": 0.06356480943581086, "rouge2_fmeasure_stderr": 0.0013703160121874625, "rouge2_precision": 0.09595929727047421, "rouge2_precision_stderr": 0.003056744064807103, "rouge2_recall": 0.060334580497235495, "rouge2_recall_stderr": 0.001371704738838684, "rougeL_fmeasure": 0.17627726645390196, "rougeL_fmeasure_stderr": 0.0020799317284272848, "rougeL_precision": 0.23221312530642865, "rougeL_precision_stderr": 0.0034637855773476654, "rougeL_recall": 0.16662869932938643, "rougeL_recall_stderr": 0.0021647677396040422, "rougeLsum_fmeasure": 0.19915256616340277, "rougeLsum_fmeasure_stderr": 0.00239939399087058, "rougeLsum_precision": 0.25797235414296243, "rougeLsum_precision_stderr": 0.00365703607288398, "rougeLsum_recall": 0.18879057223376866, "rougeLsum_recall_stderr": 0.0025014527596644433}}, "5": {"generate_text_restaurant": {"bleu": 3.487893054270047, "bleu_stderr": 0.11030761825736048, "rouge1_fmeasure": 0.257756911063572, "rouge1_fmeasure_stderr": 0.002615487581334247, "rouge1_precision": 0.331753242539869, "rouge1_precision_stderr": 0.00391429042214275, "rouge1_recall": 0.24427841616686002, "rouge1_recall_stderr": 0.002768263823097388, "rouge2_fmeasure": 0.07032602009078999, "rouge2_fmeasure_stderr": 0.0013827000891058656, "rouge2_precision": 0.10947973402783245, "rouge2_precision_stderr": 0.00326761197367824, "rouge2_recall": 0.06625511614929147, "rouge2_recall_stderr": 0.001382292688307658, "rougeL_fmeasure": 0.1923849741767262, "rougeL_fmeasure_stderr": 0.001998001966591684, "rougeL_precision": 0.25607308773270254, "rougeL_precision_stderr": 0.00353770127140282, "rougeL_recall": 0.18152620055963956, "rougeL_recall_stderr": 0.0021065308567015063, "rougeLsum_fmeasure": 0.21555168521441764, "rougeLsum_fmeasure_stderr": 0.0022919133792166186, "rougeLsum_precision": 0.2823049486328519, "rougeLsum_precision_stderr": 0.003698492501181677, "rougeLsum_recall": 0.20403990476496298, "rougeLsum_recall_stderr": 0.002418301091713723}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 0.7652139574245598, "bleu_stderr": 0.0580360191813792, "rouge1_fmeasure": 0.13284331700228394, "rouge1_fmeasure_stderr": 0.002008733661615099, "rouge1_precision": 0.09543399279052138, "rouge1_precision_stderr": 0.0015401529274997991, "rouge1_recall": 0.23270186782334634, "rouge1_recall_stderr": 0.0034796434136361463, "rouge2_fmeasure": 0.021952601017208886, "rouge2_fmeasure_stderr": 0.0009814657106866197, "rouge2_precision": 0.015354440981433308, "rouge2_precision_stderr": 0.0006868856257561475, "rouge2_recall": 0.04059213552758656, "rouge2_recall_stderr": 0.0018843299027419883, "rougeL_fmeasure": 0.11780459518526487, "rougeL_fmeasure_stderr": 0.0016583269901690515, "rougeL_precision": 0.0844375279033619, "rougeL_precision_stderr": 0.0012633659097732559, "rougeL_recall": 0.20731007297104162, "rougeL_recall_stderr": 0.0029542464284462223, "rougeLsum_fmeasure": 0.10112098884045655, "rougeLsum_fmeasure_stderr": 0.0015458255637941939, "rougeLsum_precision": 0.07242801558272945, "rougeLsum_precision_stderr": 0.0011726998663923092, "rougeLsum_recall": 0.1789651394178854, "rougeLsum_recall_stderr": 0.002834793999286031}}, "1": {"article_DOC_summary": {"bleu": 0.6234335017256344, "bleu_stderr": 0.10516892162348979, "rouge1_fmeasure": 0.12271943431877347, "rouge1_fmeasure_stderr": 0.001993408251820057, "rouge1_precision": 0.08729205756601999, "rouge1_precision_stderr": 0.0014759119556582933, "rouge1_recall": 0.2158560968459811, "rouge1_recall_stderr": 0.0033932985115285204, "rouge2_fmeasure": 0.015987682957254674, "rouge2_fmeasure_stderr": 0.0008920064252614353, "rouge2_precision": 0.011295310908611753, "rouge2_precision_stderr": 0.000637529563442929, "rouge2_recall": 0.028485623386367155, "rouge2_recall_stderr": 0.0015843238636349178, "rougeL_fmeasure": 0.10744881438263615, "rougeL_fmeasure_stderr": 0.0016582533109032562, "rougeL_precision": 0.07627791797846474, "rougeL_precision_stderr": 0.0012224414385125447, "rougeL_recall": 0.18997477453945114, "rougeL_recall_stderr": 0.002884323973027333, "rougeLsum_fmeasure": 0.09577867803568567, "rougeLsum_fmeasure_stderr": 0.0015679685291023308, "rougeLsum_precision": 0.06792038786639965, "rougeLsum_precision_stderr": 0.0011490798773779924, "rougeLsum_recall": 0.1699002900122474, "rougeLsum_recall_stderr": 0.002766652887778396}}, "2": {"article_DOC_summary": {"bleu": 0.47969693901774124, "bleu_stderr": 0.10393002306967618, "rouge1_fmeasure": 0.11490970288671254, "rouge1_fmeasure_stderr": 0.0018335229977129786, "rouge1_precision": 0.08140627832123237, "rouge1_precision_stderr": 0.0013538162390964604, "rouge1_recall": 0.20395783512684038, "rouge1_recall_stderr": 0.003180562989448791, "rouge2_fmeasure": 0.014011395875766086, "rouge2_fmeasure_stderr": 0.0007593569146491094, "rouge2_precision": 0.009805619707603953, "rouge2_precision_stderr": 0.0005371390414333858, "rouge2_recall": 0.02562908685582355, "rouge2_recall_stderr": 0.0013923986683927981, "rougeL_fmeasure": 0.10350576354209164, "rougeL_fmeasure_stderr": 0.0015393600143767505, "rougeL_precision": 0.07319916634188256, "rougeL_precision_stderr": 0.0011294377025329568, "rougeL_recall": 0.18460546669295397, "rougeL_recall_stderr": 0.00275312894855031, "rougeLsum_fmeasure": 0.08839514462271125, "rougeLsum_fmeasure_stderr": 0.0013762224046845657, "rougeLsum_precision": 0.062398343344353395, "rougeLsum_precision_stderr": 0.0010008295514121795, "rougeLsum_recall": 0.15838240344246676, "rougeLsum_recall_stderr": 0.00249154359545472}}, "3": {"article_DOC_summary": {"bleu": 0.5334095301261237, "bleu_stderr": 0.05904694475713066, "rouge1_fmeasure": 0.11336019965791448, "rouge1_fmeasure_stderr": 0.0019485384501665736, "rouge1_precision": 0.0826381581304149, "rouge1_precision_stderr": 0.0016168276430261522, "rouge1_recall": 0.1976603680004612, "rouge1_recall_stderr": 0.003302591417675845, "rouge2_fmeasure": 0.015080688670459261, "rouge2_fmeasure_stderr": 0.0007967006531578405, "rouge2_precision": 0.010818142362260436, "rouge2_precision_stderr": 0.0005875954184048923, "rouge2_recall": 0.0270483375276355, "rouge2_recall_stderr": 0.0014259889511806186, "rougeL_fmeasure": 0.10193945159447333, "rougeL_fmeasure_stderr": 0.001655352593744887, "rougeL_precision": 0.0739135150808346, "rougeL_precision_stderr": 0.0013246344100743559, "rougeL_recall": 0.1789528343212182, "rougeL_recall_stderr": 0.0028966911378406277, "rougeLsum_fmeasure": 0.08914872966484293, "rougeLsum_fmeasure_stderr": 0.0014930581583732095, "rougeLsum_precision": 0.06482067265504902, "rougeLsum_precision_stderr": 0.0012271522821360112, "rougeLsum_recall": 0.1566489179441027, "rougeLsum_recall_stderr": 0.0026261509927438605}}, "4": {"article_DOC_summary": {"bleu": 0.34520892365141886, "bleu_stderr": 0.05969467959070213, "rouge1_fmeasure": 0.03339951614864071, "rouge1_fmeasure_stderr": 0.00198021955667841, "rouge1_precision": 0.029268596464183467, "rouge1_precision_stderr": 0.0020259399835436914, "rouge1_recall": 0.05199391413839539, "rouge1_recall_stderr": 0.0030746410400925525, "rouge2_fmeasure": 0.0048903937329160254, "rouge2_fmeasure_stderr": 0.0006123327257305959, "rouge2_precision": 0.004012419731989478, "rouge2_precision_stderr": 0.0006226238446139109, "rouge2_recall": 0.007824394253170313, "rouge2_recall_stderr": 0.0009120531748140449, "rougeL_fmeasure": 0.029149042347803685, "rougeL_fmeasure_stderr": 0.0016983449338329505, "rougeL_precision": 0.025666783203699015, "rougeL_precision_stderr": 0.0017988634183294424, "rougeL_recall": 0.04584486704573437, "rougeL_recall_stderr": 0.0027083142064180553, "rougeLsum_fmeasure": 0.026316949881964886, "rougeLsum_fmeasure_stderr": 0.0015704230017045268, "rougeLsum_precision": 0.023673801967461685, "rougeLsum_precision_stderr": 0.0017485680903315058, "rougeLsum_recall": 0.04082810902018552, "rougeLsum_recall_stderr": 0.00242874515124222}}, "5": {"article_DOC_summary": {"bleu": 1.0285852034656401e-39, "bleu_stderr": 2.7605427719028158e-33, "rouge1_fmeasure": 0.002577821388984159, "rouge1_fmeasure_stderr": 0.0007153291957254458, "rouge1_precision": 0.0028853675321050755, "rouge1_precision_stderr": 0.000821656688738579, "rouge1_recall": 0.002467639720675758, "rouge1_recall_stderr": 0.000687296409234268, "rouge2_fmeasure": 0.0005725459337607456, "rouge2_fmeasure_stderr": 0.00025123537297203486, "rouge2_precision": 0.0006183705614893184, "rouge2_precision_stderr": 0.0002545551974709719, "rouge2_recall": 0.0005564448483865506, "rouge2_recall_stderr": 0.0002613253273684741, "rougeL_fmeasure": 0.002155387904801103, "rougeL_fmeasure_stderr": 0.0005925129628611262, "rougeL_precision": 0.0023823600638835174, "rougeL_precision_stderr": 0.0006598065019191697, "rougeL_recall": 0.002095069624026878, "rougeL_recall_stderr": 0.0005904857317724273, "rougeLsum_fmeasure": 0.0021943712199422225, "rougeLsum_fmeasure_stderr": 0.0006063473391893399, "rougeLsum_precision": 0.0024395355927571594, "rougeLsum_precision_stderr": 0.0006863660036715112, "rougeLsum_recall": 0.002124643173444279, "rougeLsum_recall_stderr": 0.000598494012486739}}}} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..092aaa3b53d54904443550a21309853daa5ab983 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.24113227314208618, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.028811549749581045 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.055071873695941284, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011204612925514266 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2452044168577755, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004146181231017322 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08555555702198481, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016048215948057977 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.025762355208947534, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006991588869087533 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.12033245105308268, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002914084395965689 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04031658356103043, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010315530209771078 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.054089161495901594, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010851069651214552 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.24208069073651772, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004106759426399882 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.08413720771901391, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015616400503146003 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.053253809119784244, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0010664007355544926 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.23782578142548957, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.003955768184748774 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.08276144141135114, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015265102568023873 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5eff61439bc7d2ac48c00c3c9efdb42c0b69d54e --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.14254789969047976, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.019845474922838995 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.04821268198252875, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0010579107229228407 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.232679143877019, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0038247746748433682 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.07483570357790037, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014590385445274368 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.020202864451372583, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006205320695817688 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.09387190930300064, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0024847529706070316 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03136893988102952, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009010096581650731 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.046699657881712986, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.000997164628146769 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.22598685447558764, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0036416397118735396 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07262116653857528, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013900825132693265 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.04645876164911166, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0010110982905689784 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.22267526889287845, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0036045912797825898 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07210506647702201, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0014053501287397759 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4f08c9ad78fa0445d3a94789d8891ecb40d0522d --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.1308970942789927, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.016961383512188746 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.052649460677847454, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014219603058163019 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.21440506657491165, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003827575566693626 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.07695919345861055, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015961219290256806 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02059879149840832, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007416229455782925 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.09007670334311271, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002519097222783427 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.031185481539925945, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009293186251700585 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05026535802475036, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013072494003329232 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.20834692375387437, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003703084178643004 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07398567839328438, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014846464327335155 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05039540248058442, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013436329978786829 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.20660337450142063, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0036525158438921457 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.0738855577509211, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015092163593500825 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7052825868d7e4c7d7756a7c193a0e6faf4bde80 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.14159738625676038, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.01328777336277722 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.0559017669330822, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001415504973773362 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.21020882835563734, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0037778507945221773 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08020511457184787, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016658936756924728 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02161787414155675, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007378830973683681 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.08816181362986414, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002529493308982268 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.032205510029758244, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009805578541930325 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05293674301210271, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012614159260527502 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.20363111835546907, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0036952382330166034 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.0767227253342896, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015529863044902322 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05302531776083246, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012936166080435767 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.20183908009081986, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.003615726755986769 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.0765293941605234, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015601122827046713 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..0438162757416d8776b45873c7a87455491da2f1 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.12749034254704042, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.008546817810079587 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05726731293902398, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015720916530496721 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2069366490025934, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0038164690952814466 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08088223963412505, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017198828655291519 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.021768832750612987, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007326243635228755 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.08853949896527306, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002553223455917552 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03241550862307877, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0009843107469837493 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05420226613939085, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014232892445763313 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.20092063876168395, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003735513741233921 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.07731419916529977, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015941439846477567 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05483296600876922, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014695599771991175 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.20051770757700416, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0036882494796530423 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07778405525923675, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016222838654200084 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..001773161f5f3919f8022a5dad4180a8ff9538a9 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.14672466430278214, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.01564268820920237 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06292539518254431, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017764413833392624 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.21598479779779012, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003982796672051667 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08619882119239601, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017839516781522034 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02339631866881691, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008120233797503263 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.09353320709626459, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0026698138988629928 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03440062861070681, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010372568046984475 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05887445429613583, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001577669544709431 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.20886633456863324, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003896797605122707 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.08167980777977522, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001621090928904429 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.059534473296970654, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016355653688013153 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.20802136342162436, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0037908739032240287 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.08205703397709067, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016450364163506247 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..925ae54febbefbc8265b0071b3e04ba63a55707c --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08120515498772168, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014566622323488952 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.13089295969024178, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002138790758055214 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.09183225246901366, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001446144986354779 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.007852382528498259, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0003922058129387016 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.014832775763259401, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0008057905909353507 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.00936006347003796, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00045703828961676785 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07192180042908321, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012150546841524432 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.11788475031934322, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018866313289264249 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.08185338476864322, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001222665407719195 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.07602044340370688, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013542562970553641 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.12308023723290125, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0020114849972917084 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.08606713776522688, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001341296792108946 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4746833238014895, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.033468841697744 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0c999185ecba148dc9061b463d594ccd794dda34 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.10417327087929776, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015293133070916412 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.16228858189077838, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0022341452040226693 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.11664158636209752, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015303620143841674 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.010168495209728696, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0004333723787968611 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.01767445100956555, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0009100075263152787 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.01172627441669315, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004907988758645163 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07914240123572493, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010437322612055133 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.12687740686127083, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0017053545908882837 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.08919330535243743, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010391926745202198 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.09843591549323863, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014274812071532774 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.15409355870617902, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0021165601229461333 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.11031680935504037, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001426618711627084 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.6406128660470801, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.035859594691386595 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e65297f215bee29bbef9ca86cafd40fe6148e6fc --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.0963319401570562, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014967705867028737 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.1414450916345688, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0020280581196838047 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.10401732048769743, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001405310541916511 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.007688727438441564, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00040590845811071865 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.012809419158605334, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0008177388044946671 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.008529492488429729, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004244131539762185 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07797302290028109, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011278685921172196 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.11735550235323298, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00165972064507672 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.08463427761436904, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010428084274352024 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.09051724366485044, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013894987389591678 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.13319160744817032, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0018816103018270938 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09777333618790017, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0012982359349548577 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4930700792703116, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04205732595101384 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4e3d35f6378f46dcd6570bf4b5c107c916bfa8b0 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08426872221089397, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017386132445443138 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.11184423591937163, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002071972670969457 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.08452384126554126, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014785711167470451 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.007630966876501281, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0004674881725363447 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.011142314286720276, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0007599187546842887 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.007780455376255207, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00042969979080184644 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07064490274475228, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014062280303910137 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.0961295297113538, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.001766017239716426 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.07135631553222517, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0011811827490538645 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.07886115447407943, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016407821239095097 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.10459876592461427, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0019378349486849633 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.07884453658046324, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013692319014547523 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.5575488890409951, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04792209814991415 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..65d908ea4352a0072760f7cb3a60af4764224826 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.02970493332256511, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013488671512197252 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.038478094138158066, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0016371051576469045 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.02809270781704144, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001121638874100726 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.003199741133575067, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0003382565178996562 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.004827494166756389, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0006169335309909903 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0030248237722389558, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0002667451855051004 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.025897853817423586, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011496396544348466 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.03414071229704594, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0014481845257237847 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.024641420864595455, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0009641892588452016 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.027767807703340433, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012688231361680526 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.03600781213800578, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0015355693879018626 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.0261750250556484, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0010396781819747305 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.12506380809790343, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.020376015696282453 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..363a674ba2119d610cf4df5f9259874061c91669 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.0057152840599657825, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0006591061707793925 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.007158048021481606, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0008601289654793884 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.005155523162091939, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.000539625563010443 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0006478442927818688, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0001481955341623784 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0013855253494764474, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0004196160527606927 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0006805043621601407, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0001448105863097518 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.004940457010564996, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0005670918341564018 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.006289222531320525, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0007752251249677887 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.004472796916907503, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.00046874640396691897 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.005405088855709713, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0006342299220886388 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.006747668835613718, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0008229892431308825 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.004831960704635796, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0005068548557139965 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.3355106654399238e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 4.1100081084908905e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..415788f414dc947fa81fee413b84deb31c3e0929 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.260189204618729, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.04109337902247209 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.11971887919164963, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0030700848126240347 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1617024874205441, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0025644642437620013 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.11823687062241604, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0019556352325943256 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.04054174307765198, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0019147408844450626 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.040364953765767445, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0010209689659418585 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.03213244416920702, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0009356250227933517 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.11289576504809533, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0027672672835420526 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.15853150815560624, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0025376965513659185 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.11429846594242389, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001857455051103104 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.095108940518323, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002675969804009192 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.1230176876835313, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0019806348045116966 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.09121706676453484, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015908648602271467 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..e272e7e09faeab7db96d48807f2f1a978dfa1bde --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.682192729082199, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11115643858112895 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.32428153115227637, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002645445266730213 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.2742000744658761, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0023208690537510268 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.2787366541178018, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002060360360450834 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.08120278348566318, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0016503581278321312 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.06856069612002233, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013223712649095532 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06983477129656213, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013083737751466344 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.23860589651916864, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002091926558809564 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.2010399018853146, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0017983085956935448 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.20375676150217123, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0015670128786767876 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2684559413162685, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002392978011391298 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.22555334112360417, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0020422052690831217 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.2296204517439229, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001846873774076903 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0030b74f0471601ffb3fd4026331666eaf642c8a --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.0370807194608984, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12956394178766437 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.2855909643330839, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003411790508354162 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.23383491841483692, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002863967372487143 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.24181974023597483, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0027155503537555077 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.07507496488912378, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.00195553797287265 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.05960053641050613, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001343479649823018 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06138426309539808, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013147497691957174 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.2104915776268624, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002734901496099888 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.16941974652556596, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0021320569811222414 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.17566514928441032, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020283646644480703 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.23804856738225783, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.003026140977515158 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.19311658805746856, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002453387366018241 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.20004741465910422, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0023398756942231685 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..46ba5898dabb408c05119f643f0f6e6111bc4229 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.950513026047841, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.1726499167889734 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.28523410237218244, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003745395287024571 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.22448967310033474, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0029236877569921308 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.2342323601698636, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0028075156262294494 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.08226533229108834, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0024859106665703285 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.05875737556649167, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0013832978171248258 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06106712127168939, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013548116894034825 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.21431650347706552, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.00313026915062149 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.164369322711601, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002193355190986195 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.1721492556837031, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0021172930618799423 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2396892700674045, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0033835576983395218 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.18625647639365359, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025246775303922977 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.19463119037109028, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0024298696568374094 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..dc2950070dc12b632a5489353e0de6f4d6655b08 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.171878420319261, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13165410117653997 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.3037844870072903, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003946062395029148 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.22615534444511, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0028642415459267587 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.23829866168882227, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0027541705270047426 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.09595929727047421, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.003056744064807103 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.060334580497235495, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001371704738838684 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.06356480943581086, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013703160121874625 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.23221312530642865, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0034637855773476654 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.16662869932938643, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0021647677396040422 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.17627726645390196, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020799317284272848 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.25797235414296243, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.00365703607288398 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.18879057223376866, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025014527596644433 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.19915256616340277, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.00239939399087058 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..51dfb310f986c444daca571459b09a04f39aa5bd --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 3.487893054270047, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11030761825736048 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.331753242539869, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.00391429042214275 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.24427841616686002, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002768263823097388 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.257756911063572, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002615487581334247 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.10947973402783245, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.00326761197367824 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.06625511614929147, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001382292688307658 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.07032602009078999, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0013827000891058656 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.25607308773270254, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.00353770127140282 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.18152620055963956, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0021065308567015063 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.1923849741767262, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001998001966591684 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2823049486328519, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.003698492501181677 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.20403990476496298, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002418301091713723 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.21555168521441764, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0022919133792166186 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_0.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..deb53591eec602c4b77e187457de6584590630dc --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09543399279052138, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0015401529274997991 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.23270186782334634, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0034796434136361463 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.13284331700228394, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002008733661615099 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.015354440981433308, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006868856257561475 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.04059213552758656, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0018843299027419883 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.021952601017208886, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009814657106866197 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0844375279033619, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012633659097732559 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20731007297104162, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029542464284462223 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11780459518526487, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016583269901690515 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.07242801558272945, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011726998663923092 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1789651394178854, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002834793999286031 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.10112098884045655, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015458255637941939 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.7652139574245598, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.0580360191813792 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_1.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..826b476b90b03f3ba3d5311647614876c215d2ad --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08729205756601999, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0014759119556582933 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2158560968459811, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0033932985115285204 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.12271943431877347, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.001993408251820057 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011295310908611753, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.000637529563442929 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.028485623386367155, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0015843238636349178 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.015987682957254674, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0008920064252614353 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07627791797846474, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012224414385125447 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.18997477453945114, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002884323973027333 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10744881438263615, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016582533109032562 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06792038786639965, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011490798773779924 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1699002900122474, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002766652887778396 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.09577867803568567, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015679685291023308 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6234335017256344, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10516892162348979 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_2.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..95beb9778ad71c5a26015788b1b2c3aaccdedb2a --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08140627832123237, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0013538162390964604 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.20395783512684038, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003180562989448791 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.11490970288671254, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0018335229977129786 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.009805619707603953, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005371390414333858 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.02562908685582355, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0013923986683927981 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.014011395875766086, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0007593569146491094 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07319916634188256, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0011294377025329568 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.18460546669295397, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.00275312894855031 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10350576354209164, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0015393600143767505 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.062398343344353395, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0010008295514121795 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.15838240344246676, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.00249154359545472 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.08839514462271125, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0013762224046845657 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.47969693901774124, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.10393002306967618 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_3.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..39d8c81c0bcb79085087d3cd6b7a3f30ca3021cb --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0826381581304149, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0016168276430261522 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.1976603680004612, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003302591417675845 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.11336019965791448, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0019485384501665736 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.010818142362260436, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005875954184048923 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0270483375276355, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0014259889511806186 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.015080688670459261, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0007967006531578405 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0739135150808346, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0013246344100743559 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.1789528343212182, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0028966911378406277 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10193945159447333, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001655352593744887 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06482067265504902, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0012271522821360112 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1566489179441027, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0026261509927438605 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.08914872966484293, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0014930581583732095 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.5334095301261237, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.05904694475713066 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_4.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1bf60469bb35d316e14554f70889ee4be30b2ea4 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.029268596464183467, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0020259399835436914 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.05199391413839539, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0030746410400925525 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.03339951614864071, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.00198021955667841 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.004012419731989478, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006226238446139109 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.007824394253170313, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0009120531748140449 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0048903937329160254, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0006123327257305959 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.025666783203699015, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0017988634183294424 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04584486704573437, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0027083142064180553 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.029149042347803685, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016983449338329505 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.023673801967461685, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0017485680903315058 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.04082810902018552, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.00242874515124222 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.026316949881964886, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015704230017045268 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.34520892365141886, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.05969467959070213 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_5.json b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..6955479f516b826ee4a85d030c9283cd9b39ad20 --- /dev/null +++ b/619m2b72b7/evaluation/generation/slim.619m2b72b7_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0028853675321050755, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.000821656688738579 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.002467639720675758, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.000687296409234268 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.002577821388984159, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0007153291957254458 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0006183705614893184, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0002545551974709719 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0005564448483865506, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0002613253273684741 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0005725459337607456, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00025123537297203486 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0023823600638835174, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0006598065019191697 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.002095069624026878, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0005904857317724273 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.002155387904801103, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0005925129628611262 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0024395355927571594, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0006863660036715112 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.002124643173444279, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.000598494012486739 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0021943712199422225, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0006063473391893399 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.0285852034656401e-39, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 2.7605427719028158e-33 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-2b7/619m2b72b7/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_0.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..61ed4ff8dd723ca05acffd06a55383717d5109f8 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.32,0.014758652303574886,0 +anli_r2,acc,0.316,0.014709193056057123,0 +anli_r3,acc,0.345,0.013728421539454876,0 +arc_challenge,acc,0.17235494880546076,0.011037113093461295,0 +arc_challenge,acc_norm,0.22696245733788395,0.01224049153613287,0 +arc_easy,acc,0.40025252525252525,0.010053550119896119,0 +arc_easy,acc_norm,0.367003367003367,0.009890173658452132,0 +boolq,acc,0.5559633027522936,0.008690105214920793,1 +cb,acc,0.35714285714285715,0.0646095738380922,1 +cb,f1,0.25140621178772316,,1 +copa,acc,0.57,0.04975698519562428,0 +hellaswag,acc,0.2818163712407887,0.004489648865080887,0 +hellaswag,acc_norm,0.2900816570404302,0.0045287239518782526,0 +piqa,acc,0.6224156692056583,0.011310782787145778,0 +piqa,acc_norm,0.6218715995647442,0.011313980666854535,0 +rte,acc,0.5379061371841155,0.03000984891252912,0 +sciq,acc,0.668,0.014899597242811473,0 +sciq,acc_norm,0.596,0.015524980677122581,0 +storycloze_2016,acc,0.5708177445216461,0.011445869980247972,0 +winogrande,acc,0.5177584846093133,0.014043619596174959,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_0.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c8e4eec25a1899ea5bb1bd081b24e7b9f229e328 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.32, + "acc_stderr": 0.014758652303574886 + }, + "anli_r2": { + "acc": 0.316, + "acc_stderr": 0.014709193056057123 + }, + "anli_r3": { + "acc": 0.345, + "acc_stderr": 0.013728421539454876 + }, + "cb": { + "acc": 0.35714285714285715, + "acc_stderr": 0.0646095738380922, + "f1": 0.25140621178772316 + }, + "copa": { + "acc": 0.57, + "acc_stderr": 0.04975698519562428 + }, + "hellaswag": { + "acc": 0.2818163712407887, + "acc_stderr": 0.004489648865080887, + "acc_norm": 0.2900816570404302, + "acc_norm_stderr": 0.0045287239518782526 + }, + "rte": { + "acc": 0.5379061371841155, + "acc_stderr": 0.03000984891252912 + }, + "winogrande": { + "acc": 0.5177584846093133, + "acc_stderr": 0.014043619596174959 + }, + "storycloze_2016": { + "acc": 0.5708177445216461, + "acc_stderr": 0.011445869980247972 + }, + "boolq": { + "acc": 0.5559633027522936, + "acc_stderr": 0.008690105214920793 + }, + "arc_easy": { + "acc": 0.40025252525252525, + "acc_stderr": 0.010053550119896119, + "acc_norm": 0.367003367003367, + "acc_norm_stderr": 0.009890173658452132 + }, + "arc_challenge": { + "acc": 0.17235494880546076, + "acc_stderr": 0.011037113093461295, + "acc_norm": 0.22696245733788395, + "acc_norm_stderr": 0.01224049153613287 + }, + "sciq": { + "acc": 0.668, + "acc_stderr": 0.014899597242811473, + "acc_norm": 0.596, + "acc_norm_stderr": 0.015524980677122581 + }, + "piqa": { + "acc": 0.6224156692056583, + "acc_stderr": 0.011310782787145778, + "acc_norm": 0.6218715995647442, + "acc_norm_stderr": 0.011313980666854535 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_1.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..e0d24bc4ba606e48ba2b16ac75319f277a9ab1b6 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.317,0.014721675438880219,0 +anli_r2,acc,0.335,0.014933117490932573,0 +anli_r3,acc,0.3358333333333333,0.013639261190932893,0 +arc_challenge,acc,0.17747440273037543,0.01116513876964395,0 +arc_challenge,acc_norm,0.2235494880546075,0.012174896631202607,0 +arc_easy,acc,0.37163299663299665,0.009915897123658776,0 +arc_easy,acc_norm,0.34974747474747475,0.009785578618940735,0 +boolq,acc,0.518348623853211,0.008739164562341828,1 +cb,acc,0.48214285714285715,0.06737697508644648,1 +cb,f1,0.3396316102198455,,1 +copa,acc,0.57,0.049756985195624284,0 +hellaswag,acc,0.27992431786496713,0.0044804424467629,0 +hellaswag,acc_norm,0.28510256920932087,0.004505406176606849,0 +piqa,acc,0.6213275299238302,0.011317163404516847,0 +piqa,acc_norm,0.6131664853101197,0.011363095931902852,0 +rte,acc,0.5306859205776173,0.030039730592197812,0 +sciq,acc,0.646,0.015129868238451773,0 +sciq,acc_norm,0.58,0.015615500115072957,0 +storycloze_2016,acc,0.5579903794762159,0.011484402719452579,0 +winogrande,acc,0.5059194948697711,0.014051500838485807,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_1.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_1.json new file mode 100644 index 0000000000000000000000000000000000000000..643da04057f0168fd6d677c18d6ac8baca0b969b --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.317, + "acc_stderr": 0.014721675438880219 + }, + "anli_r2": { + "acc": 0.335, + "acc_stderr": 0.014933117490932573 + }, + "anli_r3": { + "acc": 0.3358333333333333, + "acc_stderr": 0.013639261190932893 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.06737697508644648, + "f1": 0.3396316102198455 + }, + "copa": { + "acc": 0.57, + "acc_stderr": 0.049756985195624284 + }, + "hellaswag": { + "acc": 0.27992431786496713, + "acc_stderr": 0.0044804424467629, + "acc_norm": 0.28510256920932087, + "acc_norm_stderr": 0.004505406176606849 + }, + "rte": { + "acc": 0.5306859205776173, + "acc_stderr": 0.030039730592197812 + }, + "winogrande": { + "acc": 0.5059194948697711, + "acc_stderr": 0.014051500838485807 + }, + "storycloze_2016": { + "acc": 0.5579903794762159, + "acc_stderr": 0.011484402719452579 + }, + "boolq": { + "acc": 0.518348623853211, + "acc_stderr": 0.008739164562341828 + }, + "arc_easy": { + "acc": 0.37163299663299665, + "acc_stderr": 0.009915897123658776, + "acc_norm": 0.34974747474747475, + "acc_norm_stderr": 0.009785578618940735 + }, + "arc_challenge": { + "acc": 0.17747440273037543, + "acc_stderr": 0.01116513876964395, + "acc_norm": 0.2235494880546075, + "acc_norm_stderr": 0.012174896631202607 + }, + "sciq": { + "acc": 0.646, + "acc_stderr": 0.015129868238451773, + "acc_norm": 0.58, + "acc_norm_stderr": 0.015615500115072957 + }, + "piqa": { + "acc": 0.6213275299238302, + "acc_stderr": 0.011317163404516847, + "acc_norm": 0.6131664853101197, + "acc_norm_stderr": 0.011363095931902852 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_2.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..8634670b5d2e85ed6b74b540cc7a45f03820e296 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.331,0.01488827258820394,0 +anli_r2,acc,0.325,0.014818724459095526,0 +anli_r3,acc,0.3308333333333333,0.013588208070708999,0 +arc_challenge,acc,0.16723549488054607,0.010905532724601196,0 +arc_challenge,acc_norm,0.22184300341296928,0.012141659068147884,0 +arc_easy,acc,0.382996632996633,0.009974920384536479,0 +arc_easy,acc_norm,0.35353535353535354,0.00980972894815149,0 +boolq,acc,0.5223241590214067,0.008736334115585043,1 +cb,acc,0.48214285714285715,0.06737697508644648,1 +cb,f1,0.33356790992258967,,1 +copa,acc,0.56,0.04988876515698589,0 +hellaswag,acc,0.27614021111332404,0.00446173290815765,0 +hellaswag,acc_norm,0.28799044015136427,0.004519011688417175,0 +piqa,acc,0.6240478781284005,0.011301098166895727,0 +piqa,acc_norm,0.6158868335146899,0.01134816074147914,0 +rte,acc,0.5126353790613718,0.030086851767188564,0 +sciq,acc,0.633,0.015249378464171752,0 +sciq,acc_norm,0.581,0.015610338967577795,0 +storycloze_2016,acc,0.5585248530197755,0.011482952758456903,0 +winogrande,acc,0.5201262825572218,0.014041096664344334,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_2.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_2.json new file mode 100644 index 0000000000000000000000000000000000000000..990c21e201b483c676551dfbcefbf32e39be23e8 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.331, + "acc_stderr": 0.01488827258820394 + }, + "anli_r2": { + "acc": 0.325, + "acc_stderr": 0.014818724459095526 + }, + "anli_r3": { + "acc": 0.3308333333333333, + "acc_stderr": 0.013588208070708999 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.06737697508644648, + "f1": 0.33356790992258967 + }, + "copa": { + "acc": 0.56, + "acc_stderr": 0.04988876515698589 + }, + "hellaswag": { + "acc": 0.27614021111332404, + "acc_stderr": 0.00446173290815765, + "acc_norm": 0.28799044015136427, + "acc_norm_stderr": 0.004519011688417175 + }, + "rte": { + "acc": 0.5126353790613718, + "acc_stderr": 0.030086851767188564 + }, + "winogrande": { + "acc": 0.5201262825572218, + "acc_stderr": 0.014041096664344334 + }, + "storycloze_2016": { + "acc": 0.5585248530197755, + "acc_stderr": 0.011482952758456903 + }, + "boolq": { + "acc": 0.5223241590214067, + "acc_stderr": 0.008736334115585043 + }, + "arc_easy": { + "acc": 0.382996632996633, + "acc_stderr": 0.009974920384536479, + "acc_norm": 0.35353535353535354, + "acc_norm_stderr": 0.00980972894815149 + }, + "arc_challenge": { + "acc": 0.16723549488054607, + "acc_stderr": 0.010905532724601196, + "acc_norm": 0.22184300341296928, + "acc_norm_stderr": 0.012141659068147884 + }, + "sciq": { + "acc": 0.633, + "acc_stderr": 0.015249378464171752, + "acc_norm": 0.581, + "acc_norm_stderr": 0.015610338967577795 + }, + "piqa": { + "acc": 0.6240478781284005, + "acc_stderr": 0.011301098166895727, + "acc_norm": 0.6158868335146899, + "acc_norm_stderr": 0.01134816074147914 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_3.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e5aa0081a6ae7cc6b6398d68c83abaca46e78c2 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.33,0.014876872027456732,0 +anli_r2,acc,0.339,0.014976758771620344,0 +anli_r3,acc,0.32,0.013471620929769144,0 +arc_challenge,acc,0.19027303754266212,0.011470424179225688,0 +arc_challenge,acc_norm,0.2235494880546075,0.01217489663120261,0 +arc_easy,acc,0.3640572390572391,0.009873293392779117,0 +arc_easy,acc_norm,0.3463804713804714,0.009763542075695734,0 +boolq,acc,0.536085626911315,0.008722250102078081,1 +cb,acc,0.4107142857142857,0.0663363415035954,1 +cb,f1,0.286512928022362,,1 +copa,acc,0.56,0.04988876515698589,0 +hellaswag,acc,0.27902808205536744,0.004476047101806556,0 +hellaswag,acc_norm,0.2900816570404302,0.004528723951878248,0 +piqa,acc,0.6213275299238302,0.011317163404516847,0 +piqa,acc_norm,0.6109902067464635,0.011374774974447463,0 +rte,acc,0.4657039711191336,0.030025579819366422,0 +sciq,acc,0.639,0.015195720118175125,0 +sciq,acc_norm,0.587,0.01557798682993653,0 +storycloze_2016,acc,0.55264564404062,0.011498161586686652,0 +winogrande,acc,0.5122336227308603,0.014048278820405616,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_3.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_3.json new file mode 100644 index 0000000000000000000000000000000000000000..3d22036f3a67f427a550776411a1267eb2aef160 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456732 + }, + "anli_r2": { + "acc": 0.339, + "acc_stderr": 0.014976758771620344 + }, + "anli_r3": { + "acc": 0.32, + "acc_stderr": 0.013471620929769144 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.286512928022362 + }, + "copa": { + "acc": 0.56, + "acc_stderr": 0.04988876515698589 + }, + "hellaswag": { + "acc": 0.27902808205536744, + "acc_stderr": 0.004476047101806556, + "acc_norm": 0.2900816570404302, + "acc_norm_stderr": 0.004528723951878248 + }, + "rte": { + "acc": 0.4657039711191336, + "acc_stderr": 0.030025579819366422 + }, + "winogrande": { + "acc": 0.5122336227308603, + "acc_stderr": 0.014048278820405616 + }, + "storycloze_2016": { + "acc": 0.55264564404062, + "acc_stderr": 0.011498161586686652 + }, + "boolq": { + "acc": 0.536085626911315, + "acc_stderr": 0.008722250102078081 + }, + "arc_easy": { + "acc": 0.3640572390572391, + "acc_stderr": 0.009873293392779117, + "acc_norm": 0.3463804713804714, + "acc_norm_stderr": 0.009763542075695734 + }, + "arc_challenge": { + "acc": 0.19027303754266212, + "acc_stderr": 0.011470424179225688, + "acc_norm": 0.2235494880546075, + "acc_norm_stderr": 0.01217489663120261 + }, + "sciq": { + "acc": 0.639, + "acc_stderr": 0.015195720118175125, + "acc_norm": 0.587, + "acc_norm_stderr": 0.01557798682993653 + }, + "piqa": { + "acc": 0.6213275299238302, + "acc_stderr": 0.011317163404516847, + "acc_norm": 0.6109902067464635, + "acc_norm_stderr": 0.011374774974447463 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_4.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c906913c8ff02bf4b53e91e764608353b9e2aba --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.36,0.015186527932040114,0 +anli_r2,acc,0.332,0.014899597242811476,0 +anli_r3,acc,0.3408333333333333,0.013688600793296939,0 +arc_challenge,acc,0.18600682593856654,0.011370940183266736,0 +arc_challenge,acc_norm,0.2235494880546075,0.012174896631202612,0 +arc_easy,acc,0.36447811447811446,0.009875729282482438,0 +arc_easy,acc_norm,0.35269360269360267,0.009804420599378656,0 +boolq,acc,0.5382262996941896,0.00871946009810685,1 +cb,acc,0.5178571428571429,0.06737697508644647,1 +cb,f1,0.3615187514558584,,1 +copa,acc,0.6,0.04923659639173309,0 +hellaswag,acc,0.2789285002987453,0.004475557360359702,0 +hellaswag,acc_norm,0.2885879306910974,0.004521798577922144,0 +piqa,acc,0.6186071817192601,0.01133285040652868,0 +piqa,acc_norm,0.6120783460282916,0.011368965300027387,0 +rte,acc,0.48375451263537905,0.030080573208738064,0 +sciq,acc,0.63,0.015275252316519362,0 +sciq,acc_norm,0.593,0.015543249100255542,0 +storycloze_2016,acc,0.5569214323890967,0.011487262197727747,0 +winogrande,acc,0.5074980268350434,0.014050905521228577,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_4.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c051b8f7949d6eba04125f047b182742172709a2 --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.36, + "acc_stderr": 0.015186527932040114 + }, + "anli_r2": { + "acc": 0.332, + "acc_stderr": 0.014899597242811476 + }, + "anli_r3": { + "acc": 0.3408333333333333, + "acc_stderr": 0.013688600793296939 + }, + "cb": { + "acc": 0.5178571428571429, + "acc_stderr": 0.06737697508644647, + "f1": 0.3615187514558584 + }, + "copa": { + "acc": 0.6, + "acc_stderr": 0.04923659639173309 + }, + "hellaswag": { + "acc": 0.2789285002987453, + "acc_stderr": 0.004475557360359702, + "acc_norm": 0.2885879306910974, + "acc_norm_stderr": 0.004521798577922144 + }, + "rte": { + "acc": 0.48375451263537905, + "acc_stderr": 0.030080573208738064 + }, + "winogrande": { + "acc": 0.5074980268350434, + "acc_stderr": 0.014050905521228577 + }, + "storycloze_2016": { + "acc": 0.5569214323890967, + "acc_stderr": 0.011487262197727747 + }, + "boolq": { + "acc": 0.5382262996941896, + "acc_stderr": 0.00871946009810685 + }, + "arc_easy": { + "acc": 0.36447811447811446, + "acc_stderr": 0.009875729282482438, + "acc_norm": 0.35269360269360267, + "acc_norm_stderr": 0.009804420599378656 + }, + "arc_challenge": { + "acc": 0.18600682593856654, + "acc_stderr": 0.011370940183266736, + "acc_norm": 0.2235494880546075, + "acc_norm_stderr": 0.012174896631202612 + }, + "sciq": { + "acc": 0.63, + "acc_stderr": 0.015275252316519362, + "acc_norm": 0.593, + "acc_norm_stderr": 0.015543249100255542 + }, + "piqa": { + "acc": 0.6186071817192601, + "acc_stderr": 0.01133285040652868, + "acc_norm": 0.6120783460282916, + "acc_norm_stderr": 0.011368965300027387 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_5.csv b/619m2b72b7/evaluation/rankeval/619m2b72b7_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..246aa35e429693a1bc7aa4a01327f80e0cdadecc --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.344,0.015029633724408947,0 +anli_r2,acc,0.323,0.014794927843348639,0 +anli_r3,acc,0.335,0.01363087184382148,0 +arc_challenge,acc,0.18600682593856654,0.011370940183266728,0 +arc_challenge,acc_norm,0.2226962457337884,0.01215831477482993,0 +arc_easy,acc,0.3640572390572391,0.009873293392779113,0 +arc_easy,acc_norm,0.3514309764309764,0.00979639558281772,0 +boolq,acc,0.5137614678899083,0.008741742106878659,1 +cb,acc,0.42857142857142855,0.06672848092813057,1 +cb,f1,0.29855275443510737,,1 +copa,acc,0.56,0.04988876515698589,0 +hellaswag,acc,0.27833100975901215,0.004472613148508894,0 +hellaswag,acc_norm,0.2927703644692292,0.004541039698729827,0 +piqa,acc,0.6191512513601741,0.011329743189016753,0 +piqa,acc_norm,0.6164309031556039,0.01134512873411628,0 +rte,acc,0.5270758122743683,0.030052303463143706,0 +sciq,acc,0.636,0.015222868840522022,0 +sciq,acc_norm,0.611,0.01542455564730849,0 +storycloze_2016,acc,0.5515766969535008,0.011500751994277754,0 +winogrande,acc,0.5232833464877664,0.014037241309573638,0 diff --git a/619m2b72b7/evaluation/rankeval/619m2b72b7_5.json b/619m2b72b7/evaluation/rankeval/619m2b72b7_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a54599c518475dc4f2f14b6d473b4f3f5776d17d --- /dev/null +++ b/619m2b72b7/evaluation/rankeval/619m2b72b7_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.344, + "acc_stderr": 0.015029633724408947 + }, + "anli_r2": { + "acc": 0.323, + "acc_stderr": 0.014794927843348639 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.01363087184382148 + }, + "cb": { + "acc": 0.42857142857142855, + "acc_stderr": 0.06672848092813057, + "f1": 0.29855275443510737 + }, + "copa": { + "acc": 0.56, + "acc_stderr": 0.04988876515698589 + }, + "hellaswag": { + "acc": 0.27833100975901215, + "acc_stderr": 0.004472613148508894, + "acc_norm": 0.2927703644692292, + "acc_norm_stderr": 0.004541039698729827 + }, + "rte": { + "acc": 0.5270758122743683, + "acc_stderr": 0.030052303463143706 + }, + "winogrande": { + "acc": 0.5232833464877664, + "acc_stderr": 0.014037241309573638 + }, + "storycloze_2016": { + "acc": 0.5515766969535008, + "acc_stderr": 0.011500751994277754 + }, + "boolq": { + "acc": 0.5137614678899083, + "acc_stderr": 0.008741742106878659 + }, + "arc_easy": { + "acc": 0.3640572390572391, + "acc_stderr": 0.009873293392779113, + "acc_norm": 0.3514309764309764, + "acc_norm_stderr": 0.00979639558281772 + }, + "arc_challenge": { + "acc": 0.18600682593856654, + "acc_stderr": 0.011370940183266728, + "acc_norm": 0.2226962457337884, + "acc_norm_stderr": 0.01215831477482993 + }, + "sciq": { + "acc": 0.636, + "acc_stderr": 0.015222868840522022, + "acc_norm": 0.611, + "acc_norm_stderr": 0.01542455564730849 + }, + "piqa": { + "acc": 0.6191512513601741, + "acc_stderr": 0.011329743189016753, + "acc_norm": 0.6164309031556039, + "acc_norm_stderr": 0.01134512873411628 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dacd176435679433fd95cf4367308bb52e24aac --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e31890209de579c9a09f2b6504c921b2e90bb017de36ad079276e7227e9bc3 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27485f25d39199f2e730e23656a78fe17c53ca50 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a11452802875d51b6fa3030d1a6fcaaf84d7c6b4d767ac162bd73d2d6da29bd7 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e27e0ac7889afd4dffcf23417a2182f8bc0fab9d --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960ab5690e8961091ad7609eb15034cedee878665bcd19009e4baa8c192168b0 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e7323dfb6e144072190c6d13e0541d70698af96 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb008fadccb7c3b1c3e633c3ab49c0d5c23ffcdcb6fc003430009f8a92dfd943 +size 116013602 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2594b862d7a2a88504d623d2d4e1fb6442a53ef2 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f820678dfec7b5e344a0c4beefc114f5173de96860dba916b99c2062621b9211 +size 116013858 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..069a52bc0eedf28181c168bd9ca342c7ec307d3d --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a53159282b561cd980223ddad63b3fd99a286ed547ae3ceea9b0385ae0bde3 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97f5a31d5f28c75cb134b1b93529b4a9b1613a35 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030fb877d169dad9b9dd2edf7dae3438f2eac2b551f977c19993c775efb9fcb1 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6550c1b1092828c2a585bc40a9cdadcc9b0a0ca --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa344c047b440cfa73c3baecafe4af4d7b84b22090c65b3c272211539cb29776 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04d9abf305835da63625b5fc110234032df4414a --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fe915b1b02d701e900c411195d9f521e53b302dd2bf72995f085525f4fe5a5c +size 116013602 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b12700a44315f37ed9da4c3e887b242f75b5357 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95654ea8d7842d36ca1d2f21d88c286a1d1a3cd671c55100ba37b745b60bcfe5 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..922649ef3eca8df11e9523486a08f37185364622 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11581a80eacee230f934f478863f32c18319aab4a24aef85bee700878ea7bd74 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af99c772a1e429aaee1af76abcfa6a38418e61a2 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe9b9cfa176357bae30ae5d33631c2205e2be866cd89a1b621a12646a77f7ff5 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3aa828a81d09ca98a426930bde04e80bea7a839 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:585a91828c177a854a6ced14bcc29dcf8b1ed92da6a8874bacd88880e1eb7dbc +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f49c7af31210e0caba778aa3c7291500e18fd5f3 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97f21b6fa7614e376bb8fd8ec996d64d7a16b2b324cd14d2581cbd680f2b4183 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9a0f554fd6a7e4c51ca937f59e69af0d661c000 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa0356d8c742977fac2bc2786b9b334bfa83bc5f26cfc13b9ec02b96109af8e +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e744967349b09a5a6ead3c66e3c2f197f510fd4c --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5408318c8ce0fec6f4cb8200643231c35efa353057a2fdcd4eea8b97ed64eec9 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f77c216107a608692b2bc7b42c6020fa8743d91 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3bb2e48eeb9588f40ec4268abe539f84a84adb22560be4586fbf7518308ff5 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87050a0cc5a6148ccad7eaacf29d5728e4b7c22b --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53cb265797b289acea54d008bcb7e89667d814ec518d46b3eb639e7cf18fe029 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfe5c7ed635a41c763d5b6263b7081b293d23dba --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c978ce90c2451a45f71abb30a52a77062d29eea3a051151a3862ebb303ed3c +size 116013858 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e27663a8080bf16785078878d8a774ebc19e8ffc --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f3cb54f978a256f5315588ae93f7a3a07b0d5616183b18e4779eaa95f93d20 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94a3a95734df935a4d90a6c40bb90fa44264a9f3 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:929129ce753b77341da30a3ba6eb99498747e80496c593127e6ab9d079871c53 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc3d41c403704ec49e30508db3497ae79f17bc01 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8baa52936ce91cc3ade1c0ed33099aa3d80e0a5fa21aefea07f6b39ceee84f77 +size 116013602 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24def72bcdf1a3631d3477e55f629607422b3c43 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fb8823d5190fc5f00e93d4456fbd587aeebea5f3a67cd7c2143189891d5c745 +size 116013655 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61323a41cea5e32c9e51a95dd99198a614d6d9d1 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:093546aace89c7ca64d242125bb060abacd66e0613130540e2ba9257b3dc13fd +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8cf85113485787cea53abd54e720f354aa639d1 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef15d3de7a136cdc21a48281ac3cdc5b03abaa1dd139e7768aaff6a18d519904 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..576e577dce9cb96969908a66a4463e5cf46db79c --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee7710ffeb7b69905ff4cf2048d79d117c54db5403b7783edef2f65acb83360d +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6c0ce2a39b0c6c35f2e94d71774071222171de4 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6447b8a82ef52cd97c760117c05119a1d14e33612663b7a678316422f11655a2 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a275b7766223d0d4259453b54212b3e8012fc55 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7f90948b784bc99b7ae987ec1a57d5afac4ed5d6aec1ca526514f2c3efead8e +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77686b4b08f07f486b85436f1cd9a2a5de376b83 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c348a0b337a22f15510171cdf9e457ebe4d288f2edd14e75bbb1fbc51adc711 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35d07bdb8862d369029c0481e16ee850ee4564f3 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:176a95027aa6d92e47c59c0dfc385d289514f64898e8c6f2edc26f17dc2202e5 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b80b2b869c6c6cf6d2d42011d805efd59f5fe18 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e950f092b44f20221d80674515d0b924c9b781d4b10a526048f3edc3292b950 +size 116013858 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c0c4e1fd386eaf84b1bbf43afcb4fe66e2b17a --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a708b6125c2a6389de5d22bb1a5d11ff78564b5e6cc1be12bd0bb18e738b642e +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fae7a7f80586a756f8d5f4da0fb3980160362d3c --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e659a6661b17667234da2955ea92cd230fa4962fd65cce474e75bab00e35c8 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3959f9c75506fc934d728edbac9aef622acc0b32 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f825e586f5712073b729d233283b7d563f63bf0e3584e440a52465330c6cb008 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c9f7d9f97ce540405dd8d957cf80935a74e0d33 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24548d31885249991e9b2590dd4afc5041fbdf3015d364b8a5bad06bcf3685e0 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47e8b0c3cfd7b69151d2cceb81e7d250de6b3a83 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ed93df0782a472be5c2342b679d70984a43dadcc8803177bf240f73952aa6df +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c6da12b142bf863c9d43db7d30a1c94108de442 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a587f61df376a1cd4bfbb3a569d93199b2da7b960e29b29a3b2e959abd0c069 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e798f8ef0e38a42e38a998ed67aac9ad4cd987c4 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9f4b39f0c57e919fb80bdbbed0460c0a5082ff08824026ad4aeb14a9f17d37 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a063e65bd3f7c22ffb52524a8da6086e63066108 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf063fbf74623560b120e0508ecc1f6ef50855bae667268a7de308faa6e713df +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8e6f65805c4436a9e19c04a55d8132c6c3c1270 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db1b1eec6f884215abf6b9a702d2c7d2b10d65675c48b08744e04cc4c88be1d8 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efbcbbeca87ce0dc60cd758a5bd1f4d5fac73f8c --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd22d67cf2434c40f334c75c9f5249b2621470081a2aa50def0315aedfbe1b5 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..659fbff19c4267269bf00a630fe3f5f03a254056 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca526491495f26ca4b2990313b88691f3ca9a4b8331ff51e7fdc2566c2d99be9 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..970ce0c27847d2983f9edb955fbe83d4d970abc1 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f441366e11eff6c8f328297d5536b84e97d372b1a2e25d2a7ecae0f7757dfd8 +size 116013922 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7005bf93dedf464151d45cd38c494a779ed6fc0 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad50c2f2c37d6650fd538cb24aa44a7e09cce2d8117b08cea7bf4d96f976f93c +size 116013538 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..840eec477a1a2c2e89c4c65d709458dc180b6473 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc93632ef85084609a86416f203a4e2540bb827413eef8064e192b292da67d5 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dfabd796d29de856f3fda6dbf6a2061897f6271 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf0198a20826b9ba0e6d784d1b4ad40863bddb9729d0685711b86067b5790a1 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f2483beb642c5509ab764ba846a5ac5a6ed314b --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98f6ee260954e246d785e30dd0a7be34c5a2e4bc231508742cb2f86f7b77dc83 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aded1dce2577528cb0069202ccdc0bb66409f8a0 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3c8c9d4ede99b778ba34d1e1fd10f52b66dd80d0ef41937666b65a201504e2 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..052a819829ffaeccc3ddb50d2f27afd927cded15 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60dffbab84f57c9327ce8a88168a1c1a031869484f1a6f74de17ebb6251c1e44 +size 116013922 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..732870ab6607aea4e66cbabe204aa89439b357ea --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc26ce42c78bbec44f68b6ec329bbc4e2916e9d740da85d42c7d33375be72cc4 +size 116013602 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a51b35e29fad13c42046f922afa03f4b47aa6b4e --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8884b47a5867cc434f5ef5a20becd58ec7466438d36bd300b6e23eabada51293 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a60ba2a57765c18cdf6d3f6ae946b735fdc929c --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c22f1312565ae6d96826b5948d3b41517fb0d3e6dad70fc7f4216684caa45703 +size 116013602 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..262823eeee797fcabffd88ef8746c346f0fd5739 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:698ce0f1fe6e016106ca714170ae46fa7a9d8b4a64efc81599472e236702ec8c +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34ba43b0aa8733c2e9f8b689618356ad12598f38 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb24e1a49e58e96d3a9c44ea1d193e586cf54c44c0ffd9d25c35f67865a8455 +size 116013858 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3863b76581db56632e819177fb91d3722d68a2ea --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19066a2e288ec1ebf470ccacb5293b20498a5afc392377e8b5b40228f6986c01 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec3e484eea3e02dcd6d6e2f89cc3400c53fee20b --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92bdf962e9f7e2a0956cf4f2c04cfb213188bc9201bc897de9d144de7b3f4aad +size 116013655 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e56e5d91d18772dd9d93c8df1647be72620937cd --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b23501e15cad1661e23ce6cadc5a87623253824a925d5b65bcfaeb6f38592d3 +size 116013730 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac6e7561ff5822f2f8a076f86c56ee93d220f14d --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1891a24bdc8766b189b06033224cc38a61cb7df194b7c71427dfe2daab0d63d5 +size 116013794 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae2f1893f05e64b174ac6df60bed32ba904048bd --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b82952c6417b7a7f6dc0360ce0dce51fc5702ef9e42880432cdc1ac66210408 +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fe3014f8e78803ae75fb1e588ff43de111132d8 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30e9b33bfc343fb44ca42a190253b9f4d79b304047a47b42cc795e8ba440de5b +size 116013666 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49066a8490290f744c462889e7b18f203850b8fd --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3998899aff94a0ac03cb3e7bad8be14ed7bed23d319ab848fd1b812b0bed6b97 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65c8c1576e15ff72c1b5f0b1129d0ec95eb79c86 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cbcefe2f6e60da1b9bf565c957839345d39e96b15a3b7e6378e89849a907a36 +size 116013719 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b30700ae706d2b0cf0bccdfcb5ffc59f6c493f0 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b70d5fc45661e923d50246d3daca8a78c3a6dd5d42493fa5b6ac6136b7d7099 +size 116013655 diff --git a/619m2b72b7/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m2b72b7/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04c997eade506c208aaeb54aed6f2ff06940c0f7 --- /dev/null +++ b/619m2b72b7/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9e38ea57a2ec591aad4e0853ec71762b0f86361de102ac5e53c9405c1ab3d5f +size 116013591 diff --git a/619m2b72b7/global_step5111/layer_01-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d52158ca801beff0d84c4f08735d13c00b97abd1 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f65973ac6172d5864eaae23d6eb6db0e4b3e118ebd55b6eda6db72e0a9c07f +size 160826627 diff --git a/619m2b72b7/global_step5111/layer_03-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2e283482b0aa7f834a13a9d8d95b411694c5dec --- /dev/null +++ b/619m2b72b7/global_step5111/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:901af053903767ce3218ef522a7516c7205222744532fc64dd24a055443925ef +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_04-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..545d75f43cb975b6cb609a9ca7f4ba3249effbde --- /dev/null +++ b/619m2b72b7/global_step5111/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb30ba524f92da032c65a3653c5248457c676d580d4d23d995682caf733be755 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_05-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f8ce43d87cdd06b428233199b5e47e791b642f9 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87a19080e21e988282a35d760c95ad0f966aced0275916b05287126573bc6dd2 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_06-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a8e4f77c3b938474322e4fd616d7b47734dba30 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acbacf6f69c1a43c81fdca1ced3e1cce32038f29380455adafe2a6667c1586f7 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_07-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71acfaa5a40fd1c51ce31b29b6c4b94f38c02cf6 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a39b351bf1529d18e2394768168d646019a091dae322458245d487a4460748 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_08-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b6754b4ff432dbdb0529a3b0d716ef4e5f08cfb --- /dev/null +++ b/619m2b72b7/global_step5111/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29330e8f54efe976910d4cfddef1df9b378f1608fa63904cf0034bf4f01a70ac +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_09-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf4c991a513ddaf17bf3295492cc478a135f1519 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7cef80368b4f5560a8036f22b489294d937daa7e444ec2e6d8b7e82684b3b89 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_10-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c4a457c73124d8d715b56fa9a7dd200f09dbe5e --- /dev/null +++ b/619m2b72b7/global_step5111/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e462d3ba9636601c9bc094926156950ab3ccf3a69466ecd4d33b75e2df8b68 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_11-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcbcb19da191547516bafc8a7f5d9569dede9c5c --- /dev/null +++ b/619m2b72b7/global_step5111/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3161aa468af78612ac01bbd962d92448061a2a4778598a0a4667cb8157e074c +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_12-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71fbcf08f3a28739f6c5041114d6c615f2899cfd --- /dev/null +++ b/619m2b72b7/global_step5111/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c2d5823fc336be9a0b83f5ca0a6211f30322b625e6d2de3e9c7a759c0c13f8 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_13-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2852c2213554bb9c996566078989df05a0e0de55 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97339b571355562f750fe7e9b56b876fe8d8738309182a0c3d15b154a155e98 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_14-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc1fbb8a1468f2eeb1fe71b80a777ce3401ab9eb --- /dev/null +++ b/619m2b72b7/global_step5111/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c21f2e6b4307366caa6d862df9178d7fa8297f569b9472a2ec15b79e8607e1e3 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_15-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b747997c975bbc5f5487540635e5ba559aaa5e4d --- /dev/null +++ b/619m2b72b7/global_step5111/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d0de13f833c4e147056d3541656ced15f72398acdfa93bbebd3efe5374236e +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_16-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aae31ddf3cc74ac4c24a7d2dbccb9fd9964e3b07 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe3285737faee7bab2efa9e5d10edc0a56f3918c305a2259a9d68f66d20346e +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_17-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20c0f9768dcb83fbd0af8f7b5a5dce821f9f7191 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e5f319634eac2caef4efec05d9e86f2baf5453f704266095e30b20393feffe +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_18-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f155d0002ebc513468c8dad7d7cd6fa1a1b62027 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec81b5a821c268968c7b954f0d31f49842bf40847f0ac8190b86df90924769c +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_19-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99be10754d2fa3343222e185630078fa736dd80b --- /dev/null +++ b/619m2b72b7/global_step5111/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8608e6f06ae7b44b1301d847cdd442fe56db60f67ef2a9e1014cff9ffbbedf83 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_20-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3907dac1051633ef84f8ea27025989e61da84bb1 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29045c81d0353bfcd9965f2113514d0b14a2c1b53eaffed1c69064a8e70333e2 +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_21-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43745489939f472f564348d8928dd3cb4ddf6e48 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d5b81dd28d1b6ef56fc871e15da47b62c43a6a2cae4cca63489c7aa6e66c9a +size 56667395 diff --git a/619m2b72b7/global_step5111/layer_23-model_00-model_states.pt b/619m2b72b7/global_step5111/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd52f504acd74be27f6979b0c15081fb7363b6b1 --- /dev/null +++ b/619m2b72b7/global_step5111/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abee291fded1c12f6157fdd474ed8fe4c917705f2201a7b89de8a1bb923d9f2e +size 7363 diff --git a/619m2b72b7/global_step5111/mp_rank_00_model_states.pt b/619m2b72b7/global_step5111/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6aa5373ac09b2006a00a551f4a32cbf9fa80352 --- /dev/null +++ b/619m2b72b7/global_step5111/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba93847842d3812759fa72cacad4373a3a7d2e8693a0e1c21309e2e7ed3ac926 +size 38451 diff --git a/619m2b72b7/logs/2820894.err b/619m2b72b7/logs/2820894.err new file mode 100644 index 0000000000000000000000000000000000000000..60fc3957f7771724987232009ef0cf294e3d31ba --- /dev/null +++ b/619m2b72b7/logs/2820894.err @@ -0,0 +1,1113 @@ +6: 2023-02-09 22:47:59.838424: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.838434: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.838426: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.841695: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.841703: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.841732: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: 2023-02-09 22:47:59.841831: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.841839: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.841837: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.844540: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.844546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.844562: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.854844: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.854846: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.854850: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.854851: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.856829: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.859568: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.860744: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.862784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.872253: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.883549: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.886057: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.890733: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.891248: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.897884: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.909630: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.911484: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.913219: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.914591: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.924813: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.936720: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.938882: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.940110: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.941675: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 22:47:59.947440: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 22:47:59.957436: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 22:47:59.960344: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 22:47:59.963617: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:47:59.964460: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990871: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990876: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990870: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990880: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990869: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990868: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990864: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 22:47:59.990861: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999553: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999551: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999561: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999558: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999566: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999558: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999571: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 22:47:59.999550: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090529: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090540: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090544: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090548: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090535: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 22:48:00.090530: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 22:48:01.801353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:01.801814: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801825: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801829: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801829: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801832: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 22:48:01.801835: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.850988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850987: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850996: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.851002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.850997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:01.851381: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851381: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851385: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851387: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851389: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851392: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851397: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 22:48:01.851398: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:01.948902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948903: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948907: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948909: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948910: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948913: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948915: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 22:48:01.948918: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.085801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.085824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:02.086343: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086347: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086348: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086353: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086352: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086356: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086358: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 22:48:02.086361: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365084: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:02.365420: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365428: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365452: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365450: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365465: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365457: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365467: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 22:48:02.365461: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.365938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.365966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.365969: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.365991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.365984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.365989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.366011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.366010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:02.366349: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366375: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366379: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366393: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366396: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366384: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366403: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 22:48:02.366418: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397329: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:02.397650: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397652: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397685: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397688: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:02.397695: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.555638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.555654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:02.556052: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556059: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556062: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556063: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556067: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556069: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 22:48:02.556072: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 22:48:13.757592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.757614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.765159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765596: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765601: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765612: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.765614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767046: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767047: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 22:48:13.767057: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767062: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767065: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767067: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 22:48:13.767068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767435: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767452: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 22:48:13.767578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 22:48:13.767592: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.770229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.770252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772266: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772271: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772274: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772274: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 22:48:13.772403: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 22:48:13.772415: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.780725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780729: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.780745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759797: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759812: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759819: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759821: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 22:48:13.759934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 22:48:13.759949: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808856: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808878: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808892: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808891: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808894: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 22:48:13.808896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.809075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-09 22:48:13.808955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 22:48:13.808968: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.809083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809084: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.809089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810890: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810897: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810899: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810900: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810902: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 22:48:13.810944: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 22:48:13.810945: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.815526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815532: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815530: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815086: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815539: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815534: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.815534: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-09 22:48:13.815081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816970: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.816986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.816993: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 22:48:13.817036: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 22:48:13.817049: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817436: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817448: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817451: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817452: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 22:48:13.817477: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 22:48:13.817490: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +6: Successfully preprocessed all matching files. +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: +7: +7: +7: +7: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +5: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +1: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +3: Loading extension module utils... +1: Loading extension module utils... +5: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils...Loading extension module utils... +1: +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: Loading extension module utils...Loading extension module utils... +5: +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m2b72b7/logs/2820894.out b/619m2b72b7/logs/2820894.out new file mode 100644 index 0000000000000000000000000000000000000000..e6c26a70058b513a646d73caa5fe9f195f0cac9c --- /dev/null +++ b/619m2b72b7/logs/2820894.out @@ -0,0 +1,6681 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m2b72b7val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_619m2b72b7val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m2b72b7 --load checkpoints_619m2b72b7 --train-weighted-split-paths-path train2b7.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2820894.json --zero-stage 0 +START 2820894: Thu 09 Feb 2023 10:47:30 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 44.0c 102.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 38.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 49.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 39.0c 77.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 37.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 43.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 42.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 47.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +0: Launching on nid005527 (0/8), master nid005527 port 9999, GPUs 8, CUDA: True +1: Launching on nid005528 (1/8), master nid005527 port 9999, GPUs 8, CUDA: True +5: Launching on nid005532 (5/8), master nid005527 port 9999, GPUs 8, CUDA: True +3: Launching on nid005530 (3/8), master nid005527 port 9999, GPUs 8, CUDA: True +6: Launching on nid005533 (6/8), master nid005527 port 9999, GPUs 8, CUDA: True +7: Launching on nid005534 (7/8), master nid005527 port 9999, GPUs 8, CUDA: True +4: Launching on nid005531 (4/8), master nid005527 port 9999, GPUs 8, CUDA: True +2: Launching on nid005529 (2/8), master nid005527 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/2820894.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... True +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m2b72b7val +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m2b72b7 +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m2b72b7 +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m2b72b7val +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +7: > setting tensorboard ... +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-09 22:48:56,810] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.090 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 102 +0: [1/1] c++ scaled_masked_softmax_hip.o scaled_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 33.596 seconds +0: time to initialize megatron (seconds): 93.319 +0: [after megatron is initialized] datetime: 2023-02-09 22:49:33 +0: building GPT model ... +0: [2023-02-09 22:49:33,455] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-09 22:49:33,456] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-09 22:49:33,456] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.15 GB, percent = 6.0% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-09 22:49:35,467] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-09 22:49:35,698] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-09 22:49:35,698] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-09 22:49:35,698] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.19 GB, percent = 6.0% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-09 22:49:35,700] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-09 22:49:48,785] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-09 22:49:48,785] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-09 22:49:48,785] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-09 22:49:48,792] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-09 22:49:48,792] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-09 22:49:48,911] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-09 22:49:48,911] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 22:49:48,911] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.88 GB, percent = 6.1% +0: ninja: no work to do. +0: Time to load utils op: 0.1515216827392578 seconds +0: Time to load utils op: 0.1021270751953125 seconds +0: Time to load utils op: 0.20452880859375 secondsTime to load utils op: 0.20451092720031738 seconds +0: +0: Time to load utils op: 0.20418357849121094 seconds +0: Time to load utils op: 0.20448565483093262 seconds +0: Time to load utils op: 0.204498291015625 seconds +0: Time to load utils op: 0.20482969284057617 seconds +2: Time to load utils op: 0.2121448516845703 secondsTime to load utils op: 0.21209335327148438 secondsTime to load utils op: 0.21103930473327637 seconds +2: +2: +2: Time to load utils op: 0.21132206916809082 secondsTime to load utils op: 0.21135592460632324 seconds +2: +2: Time to load utils op: 0.21110248565673828 secondsTime to load utils op: 0.21125125885009766 secondsTime to load utils op: 0.21126103401184082 seconds +2: +2: +6: Time to load utils op: 0.21054339408874512 seconds +6: Time to load utils op: 0.21105527877807617 seconds +6: Time to load utils op: 0.2097485065460205 seconds +6: Time to load utils op: 0.21013736724853516 seconds +6: Time to load utils op: 0.21099448204040527 seconds +6: Time to load utils op: 0.21062970161437988 seconds +6: Time to load utils op: 0.20972013473510742 seconds +6: Time to load utils op: 0.21041369438171387 seconds +0: Time to load utils op: 0.0005764961242675781 seconds +0: Time to load utils op: 0.00035452842712402344 seconds +0: Time to load utils op: 0.0004036426544189453 seconds +0: Time to load utils op: 0.00038051605224609375 seconds +0: Time to load utils op: 0.0003714561462402344 seconds +0: Time to load utils op: 0.00039458274841308594 seconds +0: Time to load utils op: 0.0003960132598876953 seconds +2: Time to load utils op: 0.0008418560028076172 seconds +2: Time to load utils op: 0.0010721683502197266 seconds +2: Time to load utils op: 0.0011250972747802734 secondsTime to load utils op: 0.0011336803436279297 seconds +2: +2: Time to load utils op: 0.0012123584747314453 seconds +2: Time to load utils op: 0.001150369644165039 secondsTime to load utils op: 0.00118255615234375 seconds +2: +2: Time to load utils op: 0.0012240409851074219 seconds +6: Time to load utils op: 0.0009791851043701172 seconds +6: Time to load utils op: 0.0010631084442138672 seconds +6: Time to load utils op: 0.0009670257568359375 seconds +6: Time to load utils op: 0.0013434886932373047 seconds +6: Time to load utils op: 0.0013625621795654297 seconds +6: Time to load utils op: 0.0013508796691894531 secondsTime to load utils op: 0.0012829303741455078 seconds +6: +6: Time to load utils op: 0.0013508796691894531 seconds +0: [2023-02-09 22:49:49,136] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-09 22:49:49,137] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 22:49:49,137] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,253] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-09 22:49:49,253] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 22:49:49,254] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,355] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-09 22:49:49,356] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 22:49:49,356] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,458] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-09 22:49:49,459] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,459] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,559] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-09 22:49:49,559] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,560] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,663] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-09 22:49:49,663] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,664] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,775] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-09 22:49:49,776] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,776] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.04 GB, percent = 6.2% +0: [2023-02-09 22:49:49,887] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-09 22:49:49,887] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,888] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,989] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-09 22:49:49,989] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 22:49:49,989] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.03 GB, percent = 6.2% +0: [2023-02-09 22:49:49,990] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-09 22:49:49,990] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-09 22:49:49,990] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-09 22:49:49,990] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-09 22:49:49,990] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-09 22:49:49,991] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-09 22:49:49,992] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-09 22:49:49,992] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.0004286766052246094 seconds +0: [2023-02-09 22:49:49,993] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-09 22:49:50,004] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +5: Time to load utils op: 0.4141538143157959 seconds +5: Time to load utils op: 0.4141886234283447 seconds +5: Time to load utils op: 0.4141652584075928 seconds +5: Time to load utils op: 0.41419243812561035 secondsTime to load utils op: 0.4141998291015625 secondsTime to load utils op: 0.4142003059387207 seconds +5: +5: +5: Time to load utils op: 0.41421079635620117 seconds +5: Time to load utils op: 0.41420626640319824 seconds +7: Time to load utils op: 0.4146115779876709 seconds +7: Time to load utils op: 0.4146733283996582 secondsTime to load utils op: 0.414783239364624 seconds +7: +7: Time to load utils op: 0.41465044021606445 secondsTime to load utils op: 0.41465044021606445 seconds +7: +7: Time to load utils op: 0.41466736793518066 secondsTime to load utils op: 0.41466736793518066 seconds +7: +4: Time to load utils op: 0.414353609085083 seconds +7: Time to load utils op: 0.4147210121154785 seconds +4: Time to load utils op: 0.4143667221069336 secondsTime to load utils op: 0.4144010543823242 seconds +4: +4: Time to load utils op: 0.414442777633667 seconds +4: Time to load utils op: 0.4144325256347656 seconds +4: Time to load utils op: 0.4144251346588135 seconds +4: Time to load utils op: 0.4144563674926758 secondsTime to load utils op: 0.414445161819458 seconds +4: +3: Time to load utils op: 0.41483521461486816 secondsTime to load utils op: 0.4148218631744385 seconds +3: +1: Time to load utils op: 0.41494131088256836 secondsTime to load utils op: 0.41492676734924316 secondsTime to load utils op: 0.41493678092956543 seconds +1: +1: +1: Time to load utils op: 0.41490983963012695 seconds +3: Time to load utils op: 0.41484570503234863 seconds +3: Time to load utils op: 0.4148223400115967 secondsTime to load utils op: 0.4148528575897217 secondsTime to load utils op: 0.4148392677307129 seconds +3: +3: Time to load utils op: 0.41483569145202637 seconds +3: +1: Time to load utils op: 0.41500329971313477 seconds +3: Time to load utils op: 0.41489481925964355 seconds +1: Time to load utils op: 0.4149620532989502 secondsTime to load utils op: 0.41504383087158203 secondsTime to load utils op: 0.4150078296661377 seconds +1: +1: +4: Time to load utils op: 0.0009753704071044922 seconds +4: Time to load utils op: 0.0012345314025878906 seconds +4: Time to load utils op: 0.0014681816101074219 seconds +4: Time to load utils op: 0.0014562606811523438 seconds +4: Time to load utils op: 0.0014340877532958984 secondsTime to load utils op: 0.0014553070068359375 seconds +4: +4: Time to load utils op: 0.0014336109161376953 seconds +4: Time to load utils op: 0.0015120506286621094 seconds +7: Time to load utils op: 0.0007183551788330078 seconds +7: Time to load utils op: 0.0009746551513671875 seconds +7: Time to load utils op: 0.0012717247009277344 seconds +7: Time to load utils op: 0.0013387203216552734 seconds +7: Time to load utils op: 0.0012409687042236328 seconds +7: Time to load utils op: 0.0014362335205078125 seconds +7: Time to load utils op: 0.001293182373046875 seconds +7: Time to load utils op: 0.001378774642944336 seconds +3: Time to load utils op: 0.0008020401000976562 seconds +3: Time to load utils op: 0.0010461807250976562 seconds +3: Time to load utils op: 0.001188039779663086 seconds +3: Time to load utils op: 0.0013663768768310547 seconds +3: Time to load utils op: 0.00128173828125 seconds +3: Time to load utils op: 0.0013427734375 seconds +3: Time to load utils op: 0.0013735294342041016 seconds +3: Time to load utils op: 0.00144195556640625 seconds +1: Time to load utils op: 0.0006597042083740234 seconds +1: Time to load utils op: 0.0010097026824951172 seconds +1: Time to load utils op: 0.0011420249938964844 seconds +1: Time to load utils op: 0.0013184547424316406 seconds +1: Time to load utils op: 0.0012276172637939453 seconds +1: Time to load utils op: 0.0012426376342773438 secondsTime to load utils op: 0.0012106895446777344 seconds +1: +1: Time to load utils op: 0.0013127326965332031 seconds +5: Time to load utils op: 0.001081705093383789 seconds +5: Time to load utils op: 0.0012753009796142578 seconds +5: Time to load utils op: 0.001245737075805664 secondsTime to load utils op: 0.0012698173522949219 seconds +5: Time to load utils op: 0.0012984275817871094 seconds +5: +5: Time to load utils op: 0.0013344287872314453 seconds +5: Time to load utils op: 0.0013005733489990234 seconds +5: Time to load utils op: 0.0013246536254882812 seconds +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +0: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +5: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +2: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +6: [2023-02-09 22:49:51,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +4: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +3: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +5: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/mp_rank_00_model_states.pt. +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:51,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:51,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +4: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +6: [2023-02-09 22:49:52,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +3: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +7: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +5: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +2: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +0: [2023-02-09 22:49:52,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt... +1: [2023-02-09 22:49:52,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +2: [2023-02-09 22:49:52,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +7: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +5: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +0: [2023-02-09 22:49:52,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +4: [2023-02-09 22:49:52,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +1: [2023-02-09 22:49:52,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +3: [2023-02-09 22:49:52,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_01-model_00-model_states.pt. +6: [2023-02-09 22:49:52,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +4: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +2: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +7: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +3: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +6: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +5: [2023-02-09 22:49:52,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +0: [2023-02-09 22:49:52,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt... +1: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +7: [2023-02-09 22:49:52,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +2: [2023-02-09 22:49:52,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +6: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +1: [2023-02-09 22:49:52,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +4: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +0: [2023-02-09 22:49:52,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +3: [2023-02-09 22:49:52,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_03-model_00-model_states.pt. +5: [2023-02-09 22:49:52,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +4: [2023-02-09 22:49:52,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +3: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +7: [2023-02-09 22:49:52,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +5: [2023-02-09 22:49:52,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +1: [2023-02-09 22:49:52,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +6: [2023-02-09 22:49:52,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +0: [2023-02-09 22:49:52,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt... +2: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +2: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +3: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +4: [2023-02-09 22:49:52,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +5: [2023-02-09 22:49:52,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +7: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +0: [2023-02-09 22:49:52,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_04-model_00-model_states.pt. +6: [2023-02-09 22:49:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +7: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +2: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +5: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +1: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +3: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +4: [2023-02-09 22:49:52,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +6: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt... +0: [2023-02-09 22:49:52,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +4: [2023-02-09 22:49:52,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +2: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +7: [2023-02-09 22:49:52,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +3: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +0: [2023-02-09 22:49:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +1: [2023-02-09 22:49:52,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +5: [2023-02-09 22:49:52,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_05-model_00-model_states.pt. +6: [2023-02-09 22:49:52,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +4: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +3: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +5: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +6: [2023-02-09 22:49:52,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +1: [2023-02-09 22:49:52,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +7: [2023-02-09 22:49:52,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +0: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt... +2: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +2: [2023-02-09 22:49:52,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +4: [2023-02-09 22:49:52,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +3: [2023-02-09 22:49:52,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:52,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:52,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:52,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:52,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:52,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +5: [2023-02-09 22:49:52,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +6: [2023-02-09 22:49:52,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +1: [2023-02-09 22:49:52,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +0: [2023-02-09 22:49:52,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_06-model_00-model_states.pt. +7: [2023-02-09 22:49:52,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:52,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:52,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:52,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +0: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +3: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +5: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +1: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +4: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +2: [2023-02-09 22:49:53,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +6: [2023-02-09 22:49:53,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt... +7: [2023-02-09 22:49:53,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +5: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +0: [2023-02-09 22:49:53,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +3: [2023-02-09 22:49:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +2: [2023-02-09 22:49:53,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +4: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +6: [2023-02-09 22:49:53,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +1: [2023-02-09 22:49:53,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_07-model_00-model_states.pt. +7: [2023-02-09 22:49:53,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +6: [2023-02-09 22:49:53,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +7: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +3: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +0: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +2: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +5: [2023-02-09 22:49:53,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +1: [2023-02-09 22:49:53,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt... +4: [2023-02-09 22:49:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +7: [2023-02-09 22:49:53,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +3: [2023-02-09 22:49:53,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +0: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +2: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +5: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +1: [2023-02-09 22:49:53,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +4: [2023-02-09 22:49:53,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_08-model_00-model_states.pt. +6: [2023-02-09 22:49:53,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +5: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +6: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +2: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +4: [2023-02-09 22:49:53,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +3: [2023-02-09 22:49:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +1: [2023-02-09 22:49:53,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +7: [2023-02-09 22:49:53,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt... +0: [2023-02-09 22:49:53,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +6: [2023-02-09 22:49:53,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +0: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +4: [2023-02-09 22:49:53,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +3: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +2: [2023-02-09 22:49:53,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +5: [2023-02-09 22:49:53,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +1: [2023-02-09 22:49:53,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_09-model_00-model_states.pt. +7: [2023-02-09 22:49:53,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +6: [2023-02-09 22:49:53,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +4: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +1: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +3: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +5: [2023-02-09 22:49:53,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +6: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +0: [2023-02-09 22:49:53,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +7: [2023-02-09 22:49:53,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt... +2: [2023-02-09 22:49:53,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +3: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +2: [2023-02-09 22:49:53,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +1: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +5: [2023-02-09 22:49:53,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +4: [2023-02-09 22:49:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +7: [2023-02-09 22:49:53,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_10-model_00-model_states.pt. +0: [2023-02-09 22:49:53,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:53,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +6: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +1: [2023-02-09 22:49:53,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +6: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +2: [2023-02-09 22:49:53,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +7: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +3: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +0: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +5: [2023-02-09 22:49:53,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt... +4: [2023-02-09 22:49:53,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +2: [2023-02-09 22:49:53,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +4: [2023-02-09 22:49:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +1: [2023-02-09 22:49:53,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +5: [2023-02-09 22:49:53,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +7: [2023-02-09 22:49:53,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +3: [2023-02-09 22:49:53,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:53,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:53,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:53,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:53,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:53,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:53,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:53,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_11-model_00-model_states.pt. +0: [2023-02-09 22:49:53,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +6: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +4: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +3: [2023-02-09 22:49:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +2: [2023-02-09 22:49:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +5: [2023-02-09 22:49:54,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +0: [2023-02-09 22:49:54,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +1: [2023-02-09 22:49:54,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt... +7: [2023-02-09 22:49:54,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +2: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +0: [2023-02-09 22:49:54,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +3: [2023-02-09 22:49:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +6: [2023-02-09 22:49:54,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +7: [2023-02-09 22:49:54,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +5: [2023-02-09 22:49:54,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +4: [2023-02-09 22:49:54,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_12-model_00-model_states.pt. +1: [2023-02-09 22:49:54,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +6: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +5: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +0: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +3: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +2: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +1: [2023-02-09 22:49:54,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +7: [2023-02-09 22:49:54,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt... +4: [2023-02-09 22:49:54,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +5: [2023-02-09 22:49:54,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +4: [2023-02-09 22:49:54,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +2: [2023-02-09 22:49:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +3: [2023-02-09 22:49:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +0: [2023-02-09 22:49:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +1: [2023-02-09 22:49:54,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +6: [2023-02-09 22:49:54,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_13-model_00-model_states.pt. +7: [2023-02-09 22:49:54,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +4: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +5: [2023-02-09 22:49:54,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +0: [2023-02-09 22:49:54,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +2: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +7: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +6: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt... +3: [2023-02-09 22:49:54,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +4: [2023-02-09 22:49:54,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +5: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +2: [2023-02-09 22:49:54,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +3: [2023-02-09 22:49:54,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +7: [2023-02-09 22:49:54,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +1: [2023-02-09 22:49:54,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +0: [2023-02-09 22:49:54,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_14-model_00-model_states.pt. +6: [2023-02-09 22:49:54,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +6: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +5: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +3: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +7: [2023-02-09 22:49:54,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +0: [2023-02-09 22:49:54,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +2: [2023-02-09 22:49:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +1: [2023-02-09 22:49:54,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt... +4: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +4: [2023-02-09 22:49:54,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +2: [2023-02-09 22:49:54,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +5: [2023-02-09 22:49:54,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +3: [2023-02-09 22:49:54,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +7: [2023-02-09 22:49:54,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +0: [2023-02-09 22:49:54,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +6: [2023-02-09 22:49:54,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_15-model_00-model_states.pt. +1: [2023-02-09 22:49:54,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +5: [2023-02-09 22:49:54,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +5: [2023-02-09 22:49:54,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:54,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +2: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +1: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +3: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +6: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +7: [2023-02-09 22:49:54,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +0: [2023-02-09 22:49:54,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt... +4: [2023-02-09 22:49:54,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +2: [2023-02-09 22:49:54,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:54,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +4: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +3: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:54,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:54,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +6: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +1: [2023-02-09 22:49:54,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +7: [2023-02-09 22:49:54,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:54,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:54,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:54,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:54,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:54,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_16-model_00-model_states.pt. +0: [2023-02-09 22:49:54,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:54,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:54,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +5: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:55,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +7: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +2: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +6: [2023-02-09 22:49:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +3: [2023-02-09 22:49:55,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +1: [2023-02-09 22:49:55,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +0: [2023-02-09 22:49:55,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt... +4: [2023-02-09 22:49:55,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +5: [2023-02-09 22:49:55,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +4: [2023-02-09 22:49:55,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +7: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +3: [2023-02-09 22:49:55,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +6: [2023-02-09 22:49:55,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +2: [2023-02-09 22:49:55,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +1: [2023-02-09 22:49:55,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_17-model_00-model_states.pt. +0: [2023-02-09 22:49:55,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +6: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +7: [2023-02-09 22:49:55,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +3: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +0: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +5: [2023-02-09 22:49:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +4: [2023-02-09 22:49:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +1: [2023-02-09 22:49:55,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt... +2: [2023-02-09 22:49:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +4: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +2: [2023-02-09 22:49:55,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +5: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +6: [2023-02-09 22:49:55,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +7: [2023-02-09 22:49:55,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +3: [2023-02-09 22:49:55,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +0: [2023-02-09 22:49:55,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_18-model_00-model_states.pt. +1: [2023-02-09 22:49:55,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +5: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +7: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +2: [2023-02-09 22:49:55,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +3: [2023-02-09 22:49:55,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +4: [2023-02-09 22:49:55,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +6: [2023-02-09 22:49:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt... +1: [2023-02-09 22:49:55,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +4: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +2: [2023-02-09 22:49:55,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +3: [2023-02-09 22:49:55,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +5: [2023-02-09 22:49:55,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +7: [2023-02-09 22:49:55,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +6: [2023-02-09 22:49:55,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +1: [2023-02-09 22:49:55,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_19-model_00-model_states.pt. +0: [2023-02-09 22:49:55,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +4: [2023-02-09 22:49:55,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +3: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +0: [2023-02-09 22:49:55,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +1: [2023-02-09 22:49:55,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +6: [2023-02-09 22:49:55,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +2: [2023-02-09 22:49:55,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +7: [2023-02-09 22:49:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt... +5: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +4: [2023-02-09 22:49:55,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +2: [2023-02-09 22:49:55,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +0: [2023-02-09 22:49:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +3: [2023-02-09 22:49:55,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +5: [2023-02-09 22:49:55,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +6: [2023-02-09 22:49:55,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +1: [2023-02-09 22:49:55,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_20-model_00-model_states.pt. +7: [2023-02-09 22:49:55,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +4: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +3: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +5: [2023-02-09 22:49:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +1: [2023-02-09 22:49:55,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +6: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +2: [2023-02-09 22:49:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +0: [2023-02-09 22:49:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt... +7: [2023-02-09 22:49:55,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +5: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +5: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +4: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +4: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +6: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +4: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +4: [2023-02-09 22:49:55,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +2: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +1: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +1: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +0: [2023-02-09 22:49:55,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +7: [2023-02-09 22:49:55,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_21-model_00-model_states.pt. +6: [2023-02-09 22:49:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +1: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-09 22:49:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +1: [2023-02-09 22:49:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +6: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-09 22:49:55,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +7: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt... +7: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/layer_23-model_00-model_states.pt. +0: [2023-02-09 22:49:55,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +7: [2023-02-09 22:49:55,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-09 22:49:55,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +6: [2023-02-09 22:49:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,169] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +2: [2023-02-09 22:49:56,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +6: [2023-02-09 22:49:56,173] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +2: [2023-02-09 22:49:56,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +5: [2023-02-09 22:49:56,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,190] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-02-09 22:49:56,194] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +7: [2023-02-09 22:49:56,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +7: [2023-02-09 22:49:56,201] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +1: [2023-02-09 22:49:56,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,210] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +0: [2023-02-09 22:49:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +1: [2023-02-09 22:49:56,214] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +0: [2023-02-09 22:49:56,215] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +2: [2023-02-09 22:49:56,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,222] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-09 22:49:56,226] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +6: [2023-02-09 22:49:56,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,230] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +2: [2023-02-09 22:49:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,232] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +5: [2023-02-09 22:49:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +6: [2023-02-09 22:49:56,234] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +2: [2023-02-09 22:49:56,236] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +5: [2023-02-09 22:49:56,237] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +5: [2023-02-09 22:49:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,241] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +7: [2023-02-09 22:49:56,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,242] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +5: [2023-02-09 22:49:56,244] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +7: [2023-02-09 22:49:56,246] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +7: [2023-02-09 22:49:56,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,246] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +4: [2023-02-09 22:49:56,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,247] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +7: [2023-02-09 22:49:56,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +4: [2023-02-09 22:49:56,251] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +3: [2023-02-09 22:49:56,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,251] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +3: [2023-02-09 22:49:56,256] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +6: [2023-02-09 22:49:56,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,260] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +6: [2023-02-09 22:49:56,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,264] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +6: [2023-02-09 22:49:56,264] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +6: [2023-02-09 22:49:56,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +0: [2023-02-09 22:49:56,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,269] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +2: [2023-02-09 22:49:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +3: [2023-02-09 22:49:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +2: [2023-02-09 22:49:56,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +0: [2023-02-09 22:49:56,274] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +1: [2023-02-09 22:49:56,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +3: [2023-02-09 22:49:56,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +7: [2023-02-09 22:49:56,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,277] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +2: [2023-02-09 22:49:56,277] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +1: [2023-02-09 22:49:56,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +2: [2023-02-09 22:49:56,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +7: [2023-02-09 22:49:56,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +5: [2023-02-09 22:49:56,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +5: [2023-02-09 22:49:56,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +6: [2023-02-09 22:49:56,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,300] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +1: [2023-02-09 22:49:56,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +6: [2023-02-09 22:49:56,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,303] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +6: [2023-02-09 22:49:56,304] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +1: [2023-02-09 22:49:56,305] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +6: [2023-02-09 22:49:56,308] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +1: [2023-02-09 22:49:56,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,312] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +3: [2023-02-09 22:49:56,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +7: [2023-02-09 22:49:56,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,316] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +1: [2023-02-09 22:49:56,316] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +6: [2023-02-09 22:49:56,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,317] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +3: [2023-02-09 22:49:56,319] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +2: [2023-02-09 22:49:56,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,319] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +7: [2023-02-09 22:49:56,320] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +2: [2023-02-09 22:49:56,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,320] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +6: [2023-02-09 22:49:56,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +2: [2023-02-09 22:49:56,324] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +5: [2023-02-09 22:49:56,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,325] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +5: [2023-02-09 22:49:56,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +5: [2023-02-09 22:49:56,329] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +3: [2023-02-09 22:49:56,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,330] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +4: [2023-02-09 22:49:56,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,332] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +3: [2023-02-09 22:49:56,334] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +4: [2023-02-09 22:49:56,335] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +1: [2023-02-09 22:49:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +2: [2023-02-09 22:49:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-09 22:49:56,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +6: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-09 22:49:56,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +5: [2023-02-09 22:49:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +1: [2023-02-09 22:49:56,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +2: [2023-02-09 22:49:56,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +5: [2023-02-09 22:49:56,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +6: [2023-02-09 22:49:56,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +1: [2023-02-09 22:49:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,354] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +1: [2023-02-09 22:49:56,359] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +7: [2023-02-09 22:49:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,359] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +7: [2023-02-09 22:49:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,361] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +0: [2023-02-09 22:49:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +0: [2023-02-09 22:49:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +7: [2023-02-09 22:49:56,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +7: [2023-02-09 22:49:56,365] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +0: [2023-02-09 22:49:56,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +0: [2023-02-09 22:49:56,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +1: [2023-02-09 22:49:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +5: [2023-02-09 22:49:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +1: [2023-02-09 22:49:56,374] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +5: [2023-02-09 22:49:56,375] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +1: [2023-02-09 22:49:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-09 22:49:56,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-09 22:49:56,388] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +5: [2023-02-09 22:49:56,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-09 22:49:56,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +7: [2023-02-09 22:49:56,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-09 22:49:56,393] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +3: [2023-02-09 22:49:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +4: [2023-02-09 22:49:56,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +5: [2023-02-09 22:49:56,398] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +7: [2023-02-09 22:49:56,398] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +4: [2023-02-09 22:49:56,401] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +3: [2023-02-09 22:49:56,401] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +3: [2023-02-09 22:49:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +0: [2023-02-09 22:49:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,427] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +3: [2023-02-09 22:49:56,429] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +4: [2023-02-09 22:49:56,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +0: [2023-02-09 22:49:56,432] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +4: [2023-02-09 22:49:56,433] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +0: [2023-02-09 22:49:56,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,437] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +0: [2023-02-09 22:49:56,442] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +4: [2023-02-09 22:49:56,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,458] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +3: [2023-02-09 22:49:56,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,460] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +4: [2023-02-09 22:49:56,462] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +3: [2023-02-09 22:49:56,465] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +4: [2023-02-09 22:49:56,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,467] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +3: [2023-02-09 22:49:56,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-09 22:49:56,469] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +4: [2023-02-09 22:49:56,471] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +3: [2023-02-09 22:49:56,473] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +0: [2023-02-09 22:49:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +4: [2023-02-09 22:49:56,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,480] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +0: [2023-02-09 22:49:56,481] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +4: [2023-02-09 22:49:56,483] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +4: [2023-02-09 22:49:56,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-09 22:49:56,493] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +4: [2023-02-09 22:49:56,497] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +0: [2023-02-09 22:49:56,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m2b72b7/global_step5111/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-09 22:49:56,539] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +0: [2023-02-09 22:49:56,545] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +0: successfully loaded checkpoint from checkpoints_619m2b72b7 at iteration 0 +7: time (ms) | load-checkpoint: 4840.51 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-09 22:49:56 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.031628 seconds +0: number of documents: 5641150 +0: > dataset split: +0: train: +0: document indices in [0, 5641150) total of 5641150 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.095 seconds +0: total number of samples: 1317244 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.041568 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.010 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-09 22:50:10 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 23377.72 | train/valid/test-data-iterators-setup: 13041.98 +0: [after training is done] datetime: 2023-02-09 22:50:10 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 3.415532E+00 | lm loss PPL: 3.043312E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 2820894: Thu 09 Feb 2023 10:50:44 PM EET diff --git a/619m2b72b7/sbatch_619m2b72b7.sh b/619m2b72b7/sbatch_619m2b72b7.sh new file mode 100755 index 0000000000000000000000000000000000000000..964b36c40b6aa531ae693eb7866882ea8487e395 --- /dev/null +++ b/619m2b72b7/sbatch_619m2b72b7.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b72b7 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train2b7.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1_308_594 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 13_086 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b72b7/sbatch_619m2b72b7val.sh b/619m2b72b7/sbatch_619m2b72b7val.sh new file mode 100644 index 0000000000000000000000000000000000000000..6c5e613080eba7887685a450575cbe7cb4727cca --- /dev/null +++ b/619m2b72b7/sbatch_619m2b72b7val.sh @@ -0,0 +1,170 @@ +#!/bin/bash +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m2b72b7val +VARIANT_CKPT=619m2b72b7 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT +# Start from scratch +#rm -rf "$CHECKPOINT_PATH" "$TENSORBOARD_PATH" + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train2b7.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_2B7_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 2680000000 +# -> Samples: 1_308_594 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --no-load-optim \ + --reset-progress \ + --override-lr-scheduler \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m2b72b7/tensorboard_619m2b72b7/events.out.tfevents.1675875705.nid007111.101125.0 b/619m2b72b7/tensorboard_619m2b72b7/events.out.tfevents.1675875705.nid007111.101125.0 new file mode 100644 index 0000000000000000000000000000000000000000..7ea855580e057dcfbb60ba613a47e490f4237fcf --- /dev/null +++ b/619m2b72b7/tensorboard_619m2b72b7/events.out.tfevents.1675875705.nid007111.101125.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80683ab152677bea4cb76e8bcff4d84e56f12fb37648f6f15afae7ca1115884f +size 9099907 diff --git a/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675555841.nid007123.119951.0 b/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675555841.nid007123.119951.0 new file mode 100644 index 0000000000000000000000000000000000000000..6e0ef78937c0eade06132e878cadfa4620506002 --- /dev/null +++ b/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675555841.nid007123.119951.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd005224de7bfd8f4e4b4433b37c9efee215bb702859adc5e43a6d5e41f0173f +size 980 diff --git a/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675975736.nid005534.126249.0 b/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675975736.nid005534.126249.0 new file mode 100644 index 0000000000000000000000000000000000000000..575a09269eb9f99fed5b56d181fa5d4d6e250af0 --- /dev/null +++ b/619m2b72b7/tensorboard_619m2b72b7val/events.out.tfevents.1675975736.nid005534.126249.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bb7175b895d6266164cb00b5ee589115783b170f6ba5dced92bee95bffa9c71 +size 980 diff --git a/619m2b72b7/transformers/config.json b/619m2b72b7/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a1592eaab83db706537cb6afd4da42ec9828de43 --- /dev/null +++ b/619m2b72b7/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1536, "n_layer": 19, "n_head": 12, "n_inner": 6144, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/619m2b72b7/transformers/pytorch_model.bin b/619m2b72b7/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..ee157fc97ef396674afb95e36ab66ddd41ae55b8 --- /dev/null +++ b/619m2b72b7/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60bde0a366dc78daa3c010a18c200fd9cd764bc6de45d97fd61b3310710f0530 +size 1396896901