Upload 12 files

1_Pooling/config.json

@@ -1,9 +1,9 @@
 {
   "word_embedding_dimension": 1024,
   "pooling_mode_cls_token": false,
-  "pooling_mode_mean_tokens": false,
+  "pooling_mode_mean_tokens": true,
   "pooling_mode_max_tokens": false,
   "pooling_mode_mean_sqrt_len_tokens": false,
-  "pooling_mode_weightedmean_tokens": true,
+  "pooling_mode_weightedmean_tokens": false,
   "pooling_mode_lasttoken": false
 }

README.md

@@ -4,11 +4,12 @@ tags:
 - sentence-transformers
 - feature-extraction
 - sentence-similarity
+- transformers
 ---
 
 # {MODEL_NAME}
 
-This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a None dimensional dense vector space and can be used for tasks like clustering or semantic search.
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 1024 dimensional dense vector space and can be used for tasks like clustering or semantic search.
 
 <!--- Describe your model here -->
 
@@ -33,6 +34,44 @@ print(embeddings)
 
 
 
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
 ## Evaluation Results
 
 <!--- Describe how your model was evaluated -->
@@ -45,7 +84,7 @@ The model was trained with the parameters:
 
 **DataLoader**:
 
-`torch.utils.data.dataloader.DataLoader` of length 30617 with parameters:
+`torch.utils.data.dataloader.DataLoader` of length 3074 with parameters:
 ```
 {'batch_size': 16, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
 ```
@@ -57,8 +96,8 @@ The model was trained with the parameters:
 Parameters of the fit()-Method:
 ```
 {
-    "epochs": 3,
-    "evaluation_steps": 1000,
+    "epochs": 1,
+    "evaluation_steps": 500,
     "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
     "max_grad_norm": 1,
     "optimizer_class": "<class 'transformers.optimization.AdamW'>",
@@ -69,7 +108,7 @@ Parameters of the fit()-Method:
     },
     "scheduler": "WarmupLinear",
     "steps_per_epoch": null,
-    "warmup_steps": 10000,
+    "warmup_steps": 1000,
     "weight_decay": 0.01
 }
 ```
@@ -79,7 +118,7 @@ Parameters of the fit()-Method:
 ```
 SentenceTransformer(
   (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BloomModel 
-  (1): WeightedMeanPooling()
+  (1): Pooling({'word_embedding_dimension': 1024, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False})
 )
 ```
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "Mayhem50/sgpt-bloom-560M-nli",
+  "_name_or_path": "output/make-multilingual-sys-2023-02-02_16-23-11",
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
     "BloomModel"

eval/mse_evaluation_TED2020-en-fr-dev.tsv.gz_results.csv

@@ -1,55 +1,8 @@
 epoch,steps,MSE
-0,1000,1272.7741241455078
-0,2000,1239.7001266479492
-0,3000,1244.563388824463
-0,4000,1255.099868774414
-0,5000,1284.8063468933105
-0,6000,1265.7715797424316
-0,7000,1308.1769943237305
-0,8000,1277.3436546325684
-0,9000,1263.1890296936035
-0,10000,1308.1955909729004
-0,11000,1283.3735466003418
-0,12000,1262.7530097961426
-0,13000,1260.5589866638184
-0,14000,1253.4588813781738
-0,15000,1239.389419555664
-0,16000,1284.862232208252
-0,17000,1215.8602714538574
-0,18000,1208.6514472961426
-0,19000,1207.7702522277832
-0,20000,1203.3350944519043
-0,21000,1202.7355194091797
-0,22000,1193.0546760559082
-0,23000,1168.9787864685059
-0,24000,1169.53125
-0,25000,1167.289161682129
-0,26000,1183.1158638000488
-0,27000,1167.3190116882324
-0,28000,1135.9935760498047
-0,29000,1140.6807899475098
-0,30000,1136.8371963500977
-0,-1,1127.4292945861816
-1,31000,1103.3071517944336
-1,32000,1101.201057434082
-1,33000,1093.484878540039
-1,34000,1085.3593826293945
-1,35000,1087.46919631958
-1,36000,1088.6981964111328
-1,37000,1085.8448028564453
-1,38000,1079.0802955627441
-1,39000,1078.3958435058594
-1,40000,1067.711067199707
-1,41000,1070.4686164855957
-1,42000,1062.2697830200195
-1,43000,1062.1541023254395
-1,44000,1061.3683700561523
-1,45000,1068.0513381958008
-1,46000,1060.914134979248
-1,47000,1047.8565216064453
-1,48000,1049.0880012512207
-1,49000,1041.7594909667969
-1,50000,1047.9501724243164
-1,51000,1041.0179138183594
-1,52000,1047.9998588562012
-1,53000,1035.096263885498
+0,500,1047.9930877685547
+0,1000,1094.466781616211
+0,1500,1080.2101135253906
+0,2000,1067.1212196350098
+0,2500,1034.5897674560547
+0,3000,1019.6515083312988
+0,-1,1017.3401832580566

eval/translation_evaluation_TED2020-en-fr-dev.tsv.gz_results.csv

@@ -1,55 +1,8 @@
 epoch,steps,src2trg,trg2src
-0,1000,0.963,0.96
-0,2000,0.965,0.964
-0,3000,0.963,0.962
-0,4000,0.967,0.963
-0,5000,0.966,0.959
-0,6000,0.97,0.963
-0,7000,0.966,0.962
-0,8000,0.968,0.966
-0,9000,0.974,0.964
-0,10000,0.969,0.96
-0,11000,0.972,0.965
-0,12000,0.969,0.956
-0,13000,0.962,0.957
-0,14000,0.97,0.963
-0,15000,0.969,0.967
-0,16000,0.967,0.959
-0,17000,0.971,0.962
-0,18000,0.97,0.964
-0,19000,0.965,0.962
-0,20000,0.968,0.965
-0,21000,0.972,0.967
-0,22000,0.968,0.965
-0,23000,0.972,0.966
-0,24000,0.967,0.957
-0,25000,0.972,0.961
-0,26000,0.97,0.959
-0,27000,0.969,0.962
-0,28000,0.967,0.968
-0,29000,0.968,0.963
-0,30000,0.969,0.965
-0,-1,0.968,0.961
-1,31000,0.968,0.964
-1,32000,0.969,0.967
-1,33000,0.971,0.966
-1,34000,0.972,0.965
-1,35000,0.973,0.968
-1,36000,0.973,0.97
-1,37000,0.973,0.966
-1,38000,0.971,0.967
-1,39000,0.972,0.969
-1,40000,0.973,0.967
-1,41000,0.972,0.967
-1,42000,0.97,0.967
-1,43000,0.968,0.967
-1,44000,0.97,0.96
-1,45000,0.969,0.965
-1,46000,0.972,0.964
-1,47000,0.97,0.967
-1,48000,0.972,0.965
-1,49000,0.971,0.969
-1,50000,0.973,0.969
-1,51000,0.971,0.969
-1,52000,0.973,0.967
-1,53000,0.972,0.967
+0,500,0.972,0.966
+0,1000,0.971,0.97
+0,1500,0.97,0.967
+0,2000,0.968,0.964
+0,2500,0.964,0.964
+0,3000,0.968,0.962
+0,-1,0.968,0.963

modules.json

@@ -8,7 +8,7 @@
   {
     "idx": 1,
     "name": "1",
-    "path": "1_WeightedMeanPooling",
-    "type": "sentence_transformers.models.WeightedMeanPooling"
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
   }
 ]

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:785a0952ae08fc25b210abdbcb34d93157c751bf27559bd78a0a8336f4e5d902
+oid sha256:2cf41e3b230fd7cc8cf0b7223177864d8dcdcd20a199f7243ef86b563b884e8f
 size 2236953889

tokenizer_config.json

@@ -3,7 +3,7 @@
   "bos_token": "<s>",
   "eos_token": "</s>",
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "Mayhem50/sgpt-bloom-560M-nli",
+  "name_or_path": "output/make-multilingual-sys-2023-02-02_16-23-11",
   "pad_token": "<pad>",
   "padding_side": "left",
   "special_tokens_map_file": null,