Training in progress, step 214, checkpoint

checkpoint-214/1_WeightedPooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

checkpoint-214/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

checkpoint-214/config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "bobox/DeBERTa-small-ST-v1-test-step3",
+  "architectures": [
+    "DebertaV2Model"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

checkpoint-214/config_sentence_transformers.json

@@ -0,0 +1,10 @@
+{
+  "__version__": {
+    "sentence_transformers": "3.0.1",
+    "transformers": "4.44.0",
+    "pytorch": "2.4.0"
+  },
+  "prompts": {},
+  "default_prompt_name": null,
+  "similarity_fn_name": null
+}

checkpoint-214/modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_WeightedPooling",
+    "type": "__main__"
+  }
+]

checkpoint-214/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4492f30d4ededd8eb03157d61cf1ead4fa31785c5ab51705ba60639f12e733c3
+size 9475334

checkpoint-214/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1886a286545822d70cd17732b6025a52902e7429a8ab7eec28bf9c1a4f143b6c
+size 565251810

checkpoint-214/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb7e64ce619027fcced3a10efb36a72ea211f6fc84c597f456bec7808c15839
+size 14244

checkpoint-214/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:432b789d9d8239b1ae14b663527e79326efe29ce1ad7e13d9d6f9f6d522709e4
+size 1064

checkpoint-214/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

checkpoint-214/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-214/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

checkpoint-214/tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 512,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "stride": 0,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

checkpoint-214/trainer_state.json

@@ -0,0 +1,359 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.25029239766081873,
+  "eval_steps": 214,
+  "global_step": 214,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012865497076023392,
+      "grad_norm": 0.07311347126960754,
+      "learning_rate": 3.5081502480510277e-07,
+      "loss": 0.1802,
+      "step": 11
+    },
+    {
+      "epoch": 0.025730994152046785,
+      "grad_norm": 0.3663940727710724,
+      "learning_rate": 7.016300496102055e-07,
+      "loss": 0.1573,
+      "step": 22
+    },
+    {
+      "epoch": 0.03859649122807018,
+      "grad_norm": 0.22339867055416107,
+      "learning_rate": 1.0524450744153084e-06,
+      "loss": 0.1184,
+      "step": 33
+    },
+    {
+      "epoch": 0.05146198830409357,
+      "grad_norm": 0.06488268822431564,
+      "learning_rate": 1.403260099220411e-06,
+      "loss": 0.1456,
+      "step": 44
+    },
+    {
+      "epoch": 0.06432748538011696,
+      "grad_norm": 0.7134692072868347,
+      "learning_rate": 1.754075124025514e-06,
+      "loss": 0.2102,
+      "step": 55
+    },
+    {
+      "epoch": 0.07719298245614035,
+      "grad_norm": 0.048255760222673416,
+      "learning_rate": 2.104890148830617e-06,
+      "loss": 0.151,
+      "step": 66
+    },
+    {
+      "epoch": 0.09005847953216374,
+      "grad_norm": 0.43056783080101013,
+      "learning_rate": 2.4557051736357195e-06,
+      "loss": 0.1356,
+      "step": 77
+    },
+    {
+      "epoch": 0.10292397660818714,
+      "grad_norm": 0.7663404941558838,
+      "learning_rate": 2.806520198440822e-06,
+      "loss": 0.2292,
+      "step": 88
+    },
+    {
+      "epoch": 0.11578947368421053,
+      "grad_norm": 0.48333388566970825,
+      "learning_rate": 3.1573352232459252e-06,
+      "loss": 0.126,
+      "step": 99
+    },
+    {
+      "epoch": 0.1286549707602339,
+      "grad_norm": 0.5180703997612,
+      "learning_rate": 3.508150248051028e-06,
+      "loss": 0.1942,
+      "step": 110
+    },
+    {
+      "epoch": 0.1415204678362573,
+      "grad_norm": 0.6447436809539795,
+      "learning_rate": 3.85896527285613e-06,
+      "loss": 0.2089,
+      "step": 121
+    },
+    {
+      "epoch": 0.1543859649122807,
+      "grad_norm": 0.5512965321540833,
+      "learning_rate": 4.209780297661234e-06,
+      "loss": 0.1225,
+      "step": 132
+    },
+    {
+      "epoch": 0.1672514619883041,
+      "grad_norm": 0.5181564688682556,
+      "learning_rate": 4.560595322466336e-06,
+      "loss": 0.2504,
+      "step": 143
+    },
+    {
+      "epoch": 0.18011695906432748,
+      "grad_norm": 1.549516201019287,
+      "learning_rate": 4.911410347271439e-06,
+      "loss": 0.1454,
+      "step": 154
+    },
+    {
+      "epoch": 0.19298245614035087,
+      "grad_norm": 0.06744886934757233,
+      "learning_rate": 5.262225372076542e-06,
+      "loss": 0.2052,
+      "step": 165
+    },
+    {
+      "epoch": 0.20584795321637428,
+      "grad_norm": 0.3412599265575409,
+      "learning_rate": 5.613040396881644e-06,
+      "loss": 0.1321,
+      "step": 176
+    },
+    {
+      "epoch": 0.21871345029239767,
+      "grad_norm": 0.4878319203853607,
+      "learning_rate": 5.963855421686748e-06,
+      "loss": 0.1975,
+      "step": 187
+    },
+    {
+      "epoch": 0.23157894736842105,
+      "grad_norm": 0.6517124176025391,
+      "learning_rate": 6.3146704464918505e-06,
+      "loss": 0.1615,
+      "step": 198
+    },
+    {
+      "epoch": 0.24444444444444444,
+      "grad_norm": 0.1128321960568428,
+      "learning_rate": 6.665485471296953e-06,
+      "loss": 0.2174,
+      "step": 209
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_Qnli-dev_cosine_accuracy": 0.697265625,
+      "eval_Qnli-dev_cosine_accuracy_threshold": 0.6495540738105774,
+      "eval_Qnli-dev_cosine_ap": 0.7205205074888317,
+      "eval_Qnli-dev_cosine_f1": 0.6815286624203821,
+      "eval_Qnli-dev_cosine_f1_threshold": 0.4835514724254608,
+      "eval_Qnli-dev_cosine_precision": 0.5459183673469388,
+      "eval_Qnli-dev_cosine_recall": 0.9067796610169492,
+      "eval_Qnli-dev_dot_accuracy": 0.666015625,
+      "eval_Qnli-dev_dot_accuracy_threshold": 307.405517578125,
+      "eval_Qnli-dev_dot_ap": 0.676904834512821,
+      "eval_Qnli-dev_dot_f1": 0.6718027734976887,
+      "eval_Qnli-dev_dot_f1_threshold": 182.19287109375,
+      "eval_Qnli-dev_dot_precision": 0.5278450363196125,
+      "eval_Qnli-dev_dot_recall": 0.923728813559322,
+      "eval_Qnli-dev_euclidean_accuracy": 0.70703125,
+      "eval_Qnli-dev_euclidean_accuracy_threshold": 17.179746627807617,
+      "eval_Qnli-dev_euclidean_ap": 0.7311696464051982,
+      "eval_Qnli-dev_euclidean_f1": 0.6860706860706861,
+      "eval_Qnli-dev_euclidean_f1_threshold": 17.50692367553711,
+      "eval_Qnli-dev_euclidean_precision": 0.673469387755102,
+      "eval_Qnli-dev_euclidean_recall": 0.6991525423728814,
+      "eval_Qnli-dev_manhattan_accuracy": 0.703125,
+      "eval_Qnli-dev_manhattan_accuracy_threshold": 357.37890625,
+      "eval_Qnli-dev_manhattan_ap": 0.7327744127614749,
+      "eval_Qnli-dev_manhattan_f1": 0.680161943319838,
+      "eval_Qnli-dev_manhattan_f1_threshold": 373.75103759765625,
+      "eval_Qnli-dev_manhattan_precision": 0.6511627906976745,
+      "eval_Qnli-dev_manhattan_recall": 0.711864406779661,
+      "eval_Qnli-dev_max_accuracy": 0.70703125,
+      "eval_Qnli-dev_max_accuracy_threshold": 357.37890625,
+      "eval_Qnli-dev_max_ap": 0.7327744127614749,
+      "eval_Qnli-dev_max_f1": 0.6860706860706861,
+      "eval_Qnli-dev_max_f1_threshold": 373.75103759765625,
+      "eval_Qnli-dev_max_precision": 0.673469387755102,
+      "eval_Qnli-dev_max_recall": 0.923728813559322,
+      "eval_allNLI-dev_cosine_accuracy": 0.740234375,
+      "eval_allNLI-dev_cosine_accuracy_threshold": 0.7851241827011108,
+      "eval_allNLI-dev_cosine_ap": 0.6340322426598262,
+      "eval_allNLI-dev_cosine_f1": 0.6371681415929203,
+      "eval_allNLI-dev_cosine_f1_threshold": 0.5923386812210083,
+      "eval_allNLI-dev_cosine_precision": 0.5161290322580645,
+      "eval_allNLI-dev_cosine_recall": 0.8323699421965318,
+      "eval_allNLI-dev_dot_accuracy": 0.724609375,
+      "eval_allNLI-dev_dot_accuracy_threshold": 294.0272521972656,
+      "eval_allNLI-dev_dot_ap": 0.5910094855503549,
+      "eval_allNLI-dev_dot_f1": 0.5960264900662251,
+      "eval_allNLI-dev_dot_f1_threshold": 227.6922149658203,
+      "eval_allNLI-dev_dot_precision": 0.48214285714285715,
+      "eval_allNLI-dev_dot_recall": 0.7803468208092486,
+      "eval_allNLI-dev_euclidean_accuracy": 0.748046875,
+      "eval_allNLI-dev_euclidean_accuracy_threshold": 13.517601013183594,
+      "eval_allNLI-dev_euclidean_ap": 0.6422252682731053,
+      "eval_allNLI-dev_euclidean_f1": 0.65,
+      "eval_allNLI-dev_euclidean_f1_threshold": 17.501617431640625,
+      "eval_allNLI-dev_euclidean_precision": 0.5355805243445693,
+      "eval_allNLI-dev_euclidean_recall": 0.8265895953757225,
+      "eval_allNLI-dev_manhattan_accuracy": 0.74609375,
+      "eval_allNLI-dev_manhattan_accuracy_threshold": 298.3919677734375,
+      "eval_allNLI-dev_manhattan_ap": 0.6432220315530937,
+      "eval_allNLI-dev_manhattan_f1": 0.6618357487922705,
+      "eval_allNLI-dev_manhattan_f1_threshold": 358.1668701171875,
+      "eval_allNLI-dev_manhattan_precision": 0.5684647302904564,
+      "eval_allNLI-dev_manhattan_recall": 0.791907514450867,
+      "eval_allNLI-dev_max_accuracy": 0.748046875,
+      "eval_allNLI-dev_max_accuracy_threshold": 298.3919677734375,
+      "eval_allNLI-dev_max_ap": 0.6432220315530937,
+      "eval_allNLI-dev_max_f1": 0.6618357487922705,
+      "eval_allNLI-dev_max_f1_threshold": 358.1668701171875,
+      "eval_allNLI-dev_max_precision": 0.5684647302904564,
+      "eval_allNLI-dev_max_recall": 0.8323699421965318,
+      "eval_sequential_score": 0.7327744127614749,
+      "eval_sts-test_pearson_cosine": 0.8852985023833193,
+      "eval_sts-test_pearson_dot": 0.8698187375798812,
+      "eval_sts-test_pearson_euclidean": 0.9048697523934177,
+      "eval_sts-test_pearson_manhattan": 0.9054784772409876,
+      "eval_sts-test_pearson_max": 0.9054784772409876,
+      "eval_sts-test_spearman_cosine": 0.9079225861122531,
+      "eval_sts-test_spearman_dot": 0.8661416896957076,
+      "eval_sts-test_spearman_euclidean": 0.9007419777144345,
+      "eval_sts-test_spearman_manhattan": 0.9013345291665179,
+      "eval_sts-test_spearman_max": 0.9079225861122531,
+      "eval_vitaminc-pairs_loss": 2.7610974311828613,
+      "eval_vitaminc-pairs_runtime": 3.8706,
+      "eval_vitaminc-pairs_samples_per_second": 33.07,
+      "eval_vitaminc-pairs_steps_per_second": 0.258,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_negation-triplets_loss": 1.0606263875961304,
+      "eval_negation-triplets_runtime": 0.6073,
+      "eval_negation-triplets_samples_per_second": 210.757,
+      "eval_negation-triplets_steps_per_second": 1.647,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_scitail-pairs-pos_loss": 0.059617433696985245,
+      "eval_scitail-pairs-pos_runtime": 0.9151,
+      "eval_scitail-pairs-pos_samples_per_second": 139.868,
+      "eval_scitail-pairs-pos_steps_per_second": 1.093,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_scitail-pairs-qa_loss": 2.513320396246854e-07,
+      "eval_scitail-pairs-qa_runtime": 0.5906,
+      "eval_scitail-pairs-qa_samples_per_second": 216.739,
+      "eval_scitail-pairs-qa_steps_per_second": 1.693,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_xsum-pairs_loss": 0.004431054927408695,
+      "eval_xsum-pairs_runtime": 2.7873,
+      "eval_xsum-pairs_samples_per_second": 45.922,
+      "eval_xsum-pairs_steps_per_second": 0.359,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_sciq_pairs_loss": 0.013232991099357605,
+      "eval_sciq_pairs_runtime": 4.1862,
+      "eval_sciq_pairs_samples_per_second": 30.576,
+      "eval_sciq_pairs_steps_per_second": 0.239,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_qasc_pairs_loss": 0.049007147550582886,
+      "eval_qasc_pairs_runtime": 0.6288,
+      "eval_qasc_pairs_samples_per_second": 203.554,
+      "eval_qasc_pairs_steps_per_second": 1.59,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_openbookqa_pairs_loss": 1.3196656703948975,
+      "eval_openbookqa_pairs_runtime": 0.5848,
+      "eval_openbookqa_pairs_samples_per_second": 218.897,
+      "eval_openbookqa_pairs_steps_per_second": 1.71,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_msmarco_pairs_loss": 0.13587385416030884,
+      "eval_msmarco_pairs_runtime": 1.341,
+      "eval_msmarco_pairs_samples_per_second": 95.453,
+      "eval_msmarco_pairs_steps_per_second": 0.746,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_nq_pairs_loss": 0.04134061187505722,
+      "eval_nq_pairs_runtime": 2.9296,
+      "eval_nq_pairs_samples_per_second": 43.693,
+      "eval_nq_pairs_steps_per_second": 0.341,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_trivia_pairs_loss": 0.211629718542099,
+      "eval_trivia_pairs_runtime": 3.2033,
+      "eval_trivia_pairs_samples_per_second": 39.958,
+      "eval_trivia_pairs_steps_per_second": 0.312,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_gooaq_pairs_loss": 0.11869887262582779,
+      "eval_gooaq_pairs_runtime": 0.869,
+      "eval_gooaq_pairs_samples_per_second": 147.291,
+      "eval_gooaq_pairs_steps_per_second": 1.151,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_paws-pos_loss": 0.04590649530291557,
+      "eval_paws-pos_runtime": 0.6652,
+      "eval_paws-pos_samples_per_second": 192.433,
+      "eval_paws-pos_steps_per_second": 1.503,
+      "step": 214
+    },
+    {
+      "epoch": 0.25029239766081873,
+      "eval_global_dataset_loss": 0.21873071789741516,
+      "eval_global_dataset_runtime": 24.2472,
+      "eval_global_dataset_samples_per_second": 27.343,
+      "eval_global_dataset_steps_per_second": 0.247,
+      "step": 214
+    }
+  ],
+  "logging_steps": 11,
+  "max_steps": 4275,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 214,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 96,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-214/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aed6626d5e4516eecaff94019f86ab02eb588df865930f776737c69908518829
+size 5624