init commit

README.md

@@ -0,0 +1,40 @@
+This is finetune version of [SimCSE: Simple Contrastive Learning of Sentence Embeddings](https://arxiv.org/abs/2104.08821)
+, train unsupervised on 100K triplet samples samples related to stroke from : stroke books, quora medical, quora's stroke and human annotates.
+
+### Extract sentence representation
+```
+from transformers import AutoTokenizer, AutoModel  
+tokenizer = AutoTokenizer.from_pretrained("demdecuong/stroke_simcse")
+model = AutoModel.from_pretrained("demdecuong/stroke_simcse")
+
+text = "What are disease related to red stroke's causes?"
+inputs = tokenizer(text, return_tensors='pt')
+outputs = model(**inputs)[1]
+```
+### Build up embedding for database
+
+```
+database = [
+    'What is the daily checklist for stroke returning home',
+    'What are some tips for stroke adapt new life',
+    'What  should I consider when using nursing-home care'
+]
+
+embedding = torch.zeros((len(database),768))
+
+for i in range(len(database)):
+  inputs = tokenizer(database[i], return_tensors="pt")
+  outputs = model(**inputs)[1]
+  embedding[i] = outputs
+
+print(embedding.shape)
+```
+
+### Result
+On our company's PoC project, the testset contains positive/negative pairs of matching question related to stroke from human-generation. 
+
+| Model  | Top-1 Accuracy |
+| ------------- | ------------- |
+| SimCSE (supervised)  | 75.83  |
+| SimCSE unsupervised (ours)  | 76.66  |
+| SimCSE supervised (ours)  | TBD  |

config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "result/my-sup-simcse-bert-base-uncased",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.2.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

eval_results.txt

@@ -0,0 +1,12 @@
+epoch = 3.0
+eval_CR = 89.02
+eval_MPQA = 89.09
+eval_MR = 83.33
+eval_MRPC = 75.17
+eval_SST2 = 85.67
+eval_SUBJ = 99.65
+eval_TREC = 82.72
+eval_avg_sts = 0.8388995317355574
+eval_avg_transfer = 86.37857142857142
+eval_sickr_spearman = 0.8146857710857348
+eval_stsb_spearman = 0.86311329238538

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f877c353b81f010c67f366c2ab146aadc60349440f54c2b9e4ae87f38743bbe8
+size 875982637

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0988e26f9c1e96651f452af503bba057ef2c9cc9b9f364ef9ad28e26d9af9c59
+size 438000759

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f676bf30734a5585dd13ee674039589cb4b9ca0275c962b5e11a3b02a83586a6
+size 623

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "name_or_path": "princeton-nlp/sup-simcse-bert-base-uncased", "special_tokens_map_file": "/root/.cache/huggingface/transformers/8c406286308d13c3a53bc10c3d1a2d5113d4e46a34cb6ec5ee06e5d9762c462c.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d", "do_basic_tokenize": true, "never_split": null}

train_results.txt

@@ -0,0 +1,3 @@
+epoch = 3.0
+train_runtime = 3382.225
+train_samples_per_second = 0.716

trainer_state.json

@@ -0,0 +1,179 @@
+{
+  "best_metric": 0.86311329238538,
+  "best_model_checkpoint": "result/my-sup-simcse-bert-base-uncased",
+  "epoch": 3.0,
+  "global_step": 2421,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.15,
+      "eval_avg_sts": 0.8388995317355574,
+      "eval_sickr_spearman": 0.8146857710857348,
+      "eval_stsb_spearman": 0.86311329238538,
+      "step": 125
+    },
+    {
+      "epoch": 0.31,
+      "eval_avg_sts": 0.8332486405485358,
+      "eval_sickr_spearman": 0.8082352902247991,
+      "eval_stsb_spearman": 0.8582619908722724,
+      "step": 250
+    },
+    {
+      "epoch": 0.46,
+      "eval_avg_sts": 0.8315926314982992,
+      "eval_sickr_spearman": 0.8008793270395521,
+      "eval_stsb_spearman": 0.8623059359570463,
+      "step": 375
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.9673688558446925e-05,
+      "loss": 0.1625,
+      "step": 500
+    },
+    {
+      "epoch": 0.62,
+      "eval_avg_sts": 0.82902006063027,
+      "eval_sickr_spearman": 0.8009135732148775,
+      "eval_stsb_spearman": 0.8571265480456624,
+      "step": 500
+    },
+    {
+      "epoch": 0.77,
+      "eval_avg_sts": 0.8234837706323759,
+      "eval_sickr_spearman": 0.7913043269678645,
+      "eval_stsb_spearman": 0.8556632142968873,
+      "step": 625
+    },
+    {
+      "epoch": 0.93,
+      "eval_avg_sts": 0.8238431555715283,
+      "eval_sickr_spearman": 0.7974856415363409,
+      "eval_stsb_spearman": 0.8502006696067157,
+      "step": 750
+    },
+    {
+      "epoch": 1.08,
+      "eval_avg_sts": 0.8266718232644965,
+      "eval_sickr_spearman": 0.8003919554532598,
+      "eval_stsb_spearman": 0.8529516910757333,
+      "step": 875
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 2.9347377116893847e-05,
+      "loss": 0.0815,
+      "step": 1000
+    },
+    {
+      "epoch": 1.24,
+      "eval_avg_sts": 0.819954267975217,
+      "eval_sickr_spearman": 0.7929691810059123,
+      "eval_stsb_spearman": 0.8469393549445217,
+      "step": 1000
+    },
+    {
+      "epoch": 1.39,
+      "eval_avg_sts": 0.8249139721135836,
+      "eval_sickr_spearman": 0.8003723587638729,
+      "eval_stsb_spearman": 0.8494555854632942,
+      "step": 1125
+    },
+    {
+      "epoch": 1.55,
+      "eval_avg_sts": 0.8211704289881193,
+      "eval_sickr_spearman": 0.794750558496049,
+      "eval_stsb_spearman": 0.8475902994801896,
+      "step": 1250
+    },
+    {
+      "epoch": 1.7,
+      "eval_avg_sts": 0.8210084871203903,
+      "eval_sickr_spearman": 0.7898188690936879,
+      "eval_stsb_spearman": 0.8521981051470927,
+      "step": 1375
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 1.902106567534077e-05,
+      "loss": 0.0673,
+      "step": 1500
+    },
+    {
+      "epoch": 1.86,
+      "eval_avg_sts": 0.8244940569196582,
+      "eval_sickr_spearman": 0.7977014933062037,
+      "eval_stsb_spearman": 0.8512866205331128,
+      "step": 1500
+    },
+    {
+      "epoch": 2.01,
+      "eval_avg_sts": 0.819840729451552,
+      "eval_sickr_spearman": 0.789596437062928,
+      "eval_stsb_spearman": 0.850085021840176,
+      "step": 1625
+    },
+    {
+      "epoch": 2.17,
+      "eval_avg_sts": 0.8194523296306793,
+      "eval_sickr_spearman": 0.7900231453681041,
+      "eval_stsb_spearman": 0.8488815138932543,
+      "step": 1750
+    },
+    {
+      "epoch": 2.32,
+      "eval_avg_sts": 0.819201881270627,
+      "eval_sickr_spearman": 0.7891284700416165,
+      "eval_stsb_spearman": 0.8492752924996376,
+      "step": 1875
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 8.694754233787691e-06,
+      "loss": 0.0613,
+      "step": 2000
+    },
+    {
+      "epoch": 2.48,
+      "eval_avg_sts": 0.8205549448908287,
+      "eval_sickr_spearman": 0.7917587011874689,
+      "eval_stsb_spearman": 0.8493511885941886,
+      "step": 2000
+    },
+    {
+      "epoch": 2.63,
+      "eval_avg_sts": 0.8199800089125161,
+      "eval_sickr_spearman": 0.7909771871359695,
+      "eval_stsb_spearman": 0.8489828306890627,
+      "step": 2125
+    },
+    {
+      "epoch": 2.79,
+      "eval_avg_sts": 0.8217889587200623,
+      "eval_sickr_spearman": 0.7920187896017563,
+      "eval_stsb_spearman": 0.8515591278383683,
+      "step": 2250
+    },
+    {
+      "epoch": 2.94,
+      "eval_avg_sts": 0.8220433030575918,
+      "eval_sickr_spearman": 0.7925120690135258,
+      "eval_stsb_spearman": 0.8515745371016578,
+      "step": 2375
+    },
+    {
+      "epoch": 3.0,
+      "step": 2421,
+      "train_runtime": 3382.225,
+      "train_samples_per_second": 0.716
+    }
+  ],
+  "max_steps": 2421,
+  "num_train_epochs": 3,
+  "total_flos": 19636662129721344,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c57010aba088aae779a723a1b3d42d330743938195c9820bfb96be9122bd6d03
+size 2095