Add SetFit model

1_Pooling/config.json

@@ -1,5 +1,5 @@
 {
-  "word_embedding_dimension": 768,
+  "word_embedding_dimension": 384,
   "pooling_mode_cls_token": false,
   "pooling_mode_mean_tokens": true,
   "pooling_mode_max_tokens": false,

README.md

@@ -5,7 +5,7 @@ tags:
 - sentence-transformers
 - text-classification
 - generated_from_setfit_trainer
-base_model: sentence-transformers/paraphrase-mpnet-base-v2
+base_model: sentence-transformers/paraphrase-MiniLM-L6-v2
 metrics:
 - accuracy
 widget:
@@ -69,9 +69,9 @@ pipeline_tag: text-classification
 inference: false
 ---
 
-# SetFit with sentence-transformers/paraphrase-mpnet-base-v2
+# SetFit with sentence-transformers/paraphrase-MiniLM-L6-v2
 
-This is a [SetFit](https://github.com/huggingface/setfit) model that can be used for Text Classification. This SetFit model uses [sentence-transformers/paraphrase-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-mpnet-base-v2) as the Sentence Transformer embedding model. A OneVsRestClassifier instance is used for classification.
+This is a [SetFit](https://github.com/huggingface/setfit) model that can be used for Text Classification. This SetFit model uses [sentence-transformers/paraphrase-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/paraphrase-MiniLM-L6-v2) as the Sentence Transformer embedding model. A OneVsRestClassifier instance is used for classification.
 
 The model has been trained using an efficient few-shot learning technique that involves:
 
@@ -82,9 +82,9 @@ The model has been trained using an efficient few-shot learning technique that i
 
 ### Model Description
 - **Model Type:** SetFit
-- **Sentence Transformer body:** [sentence-transformers/paraphrase-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-mpnet-base-v2)
+- **Sentence Transformer body:** [sentence-transformers/paraphrase-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/paraphrase-MiniLM-L6-v2)
 - **Classification head:** a OneVsRestClassifier instance
-- **Maximum Sequence Length:** 512 tokens
+- **Maximum Sequence Length:** 128 tokens
 <!-- - **Number of Classes:** Unknown -->
 <!-- - **Training Dataset:** [Unknown](https://huggingface.co/datasets/unknown) -->
 <!-- - **Language:** Unknown -->
@@ -170,20 +170,69 @@ preds = model("Seminar Internet Technology Das Seminar behandelt aktuelle Themen
 ### Training Results
 | Epoch  | Step | Training Loss | Validation Loss |
 |:------:|:----:|:-------------:|:---------------:|
-| 0.0014 | 1    | 0.2798        | -               |
-| 0.0716 | 50   | 0.2227        | -               |
-| 0.1433 | 100  | 0.1775        | -               |
-| 0.2149 | 150  | 0.1471        | -               |
-| 0.2865 | 200  | 0.1325        | -               |
-| 0.3582 | 250  | 0.1292        | -               |
-| 0.4298 | 300  | 0.1067        | -               |
-| 0.5014 | 350  | 0.0985        | -               |
-| 0.5731 | 400  | 0.0943        | -               |
-| 0.6447 | 450  | 0.091         | -               |
-| 0.7163 | 500  | 0.0786        | -               |
-| 0.7880 | 550  | 0.0911        | -               |
-| 0.8596 | 600  | 0.0762        | -               |
-| 0.9312 | 650  | 0.0775        | -               |
+| 0.0014 | 1    | 0.3334        | -               |
+| 0.0716 | 50   | 0.2411        | -               |
+| 0.1433 | 100  | 0.2124        | -               |
+| 0.2149 | 150  | 0.186         | -               |
+| 0.2865 | 200  | 0.1806        | -               |
+| 0.3582 | 250  | 0.1759        | -               |
+| 0.4298 | 300  | 0.1705        | -               |
+| 0.5014 | 350  | 0.1542        | -               |
+| 0.5731 | 400  | 0.1559        | -               |
+| 0.6447 | 450  | 0.1524        | -               |
+| 0.7163 | 500  | 0.1438        | -               |
+| 0.7880 | 550  | 0.1507        | -               |
+| 0.8596 | 600  | 0.14          | -               |
+| 0.9312 | 650  | 0.1466        | -               |
+| 0.0006 | 1    | 0.1157        | -               |
+| 0.0287 | 50   | 0.1266        | -               |
+| 0.0573 | 100  | 0.1325        | -               |
+| 0.0860 | 150  | 0.1237        | -               |
+| 0.1147 | 200  | 0.12          | -               |
+| 0.1433 | 250  | 0.1189        | -               |
+| 0.1720 | 300  | 0.1094        | -               |
+| 0.2007 | 350  | 0.1028        | -               |
+| 0.2294 | 400  | 0.0993        | -               |
+| 0.2580 | 450  | 0.1003        | -               |
+| 0.2867 | 500  | 0.0898        | -               |
+| 0.3154 | 550  | 0.0875        | -               |
+| 0.3440 | 600  | 0.0847        | -               |
+| 0.3727 | 650  | 0.0879        | -               |
+| 0.4014 | 700  | 0.0801        | -               |
+| 0.4300 | 750  | 0.0754        | -               |
+| 0.4587 | 800  | 0.0791        | -               |
+| 0.4874 | 850  | 0.0715        | -               |
+| 0.5161 | 900  | 0.0781        | -               |
+| 0.5447 | 950  | 0.0765        | -               |
+| 0.5734 | 1000 | 0.0718        | -               |
+| 0.6021 | 1050 | 0.0786        | -               |
+| 0.6307 | 1100 | 0.073         | -               |
+| 0.6594 | 1150 | 0.0705        | -               |
+| 0.6881 | 1200 | 0.072         | -               |
+| 0.7167 | 1250 | 0.0673        | -               |
+| 0.7454 | 1300 | 0.066         | -               |
+| 0.7741 | 1350 | 0.0671        | -               |
+| 0.8028 | 1400 | 0.0631        | -               |
+| 0.8314 | 1450 | 0.0673        | -               |
+| 0.8601 | 1500 | 0.0638        | -               |
+| 0.8888 | 1550 | 0.0674        | -               |
+| 0.9174 | 1600 | 0.0613        | -               |
+| 0.9461 | 1650 | 0.063         | -               |
+| 0.9748 | 1700 | 0.0682        | -               |
+| 0.0014 | 1    | 0.0497        | -               |
+| 0.0716 | 50   | 0.0584        | -               |
+| 0.1433 | 100  | 0.0663        | -               |
+| 0.2149 | 150  | 0.0682        | -               |
+| 0.2865 | 200  | 0.0616        | -               |
+| 0.3582 | 250  | 0.0657        | -               |
+| 0.4298 | 300  | 0.0593        | -               |
+| 0.5014 | 350  | 0.0593        | -               |
+| 0.5731 | 400  | 0.0565        | -               |
+| 0.6447 | 450  | 0.0595        | -               |
+| 0.7163 | 500  | 0.0589        | -               |
+| 0.7880 | 550  | 0.0649        | -               |
+| 0.8596 | 600  | 0.0554        | -               |
+| 0.9312 | 650  | 0.0601        | -               |
 
 ### Framework Versions
 - Python: 3.12.3

config.json

@@ -1,24 +1,26 @@
 {
-  "_name_or_path": "sentence-transformers/paraphrase-mpnet-base-v2",
+  "_name_or_path": "sentence-transformers/paraphrase-MiniLM-L6-v2",
   "architectures": [
-    "MPNetModel"
+    "BertModel"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
-  "eos_token_id": 2,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
+  "hidden_size": 384,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 514,
-  "model_type": "mpnet",
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 1,
-  "relative_attention_num_buckets": 32,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
   "torch_dtype": "float32",
   "transformers_version": "4.43.1",
-  "vocab_size": 30527
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92972fdc0601d168d56004e78d57ba956c481075c9ea4613576d48e38177b8e9
-size 437967672
+oid sha256:db084c914530afe8da5c4c9ca8188cf25f0f6947c28be54680fb375c036b64b4
+size 90864192

model_head.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3631581829b8a144cf4ed104775763cd8f15d7c6018e93f041e2edb943bdd511
-size 39828
+oid sha256:3c4dcf0c31745b0af1e0164490cc79d5e24b8d09e028a744b5b5f2258404271f
+size 21396

sentence_bert_config.json

@@ -1,4 +1,4 @@
 {
-  "max_seq_length": 512,
+  "max_seq_length": 128,
   "do_lower_case": false
 }

special_tokens_map.json

@@ -1,41 +1,27 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
+    "content": "[CLS]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "mask_token": {
-    "content": "<mask>",
-    "lstrip": true,
+    "content": "[MASK]",
+    "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<pad>",
+    "content": "[PAD]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "sep_token": {
-    "content": "</s>",
+    "content": "[SEP]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -1,59 +1,57 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "<s>",
+      "content": "[PAD]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1": {
-      "content": "<pad>",
+    "100": {
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "2": {
-      "content": "</s>",
+    "101": {
+      "content": "[CLS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "104": {
-      "content": "[UNK]",
+    "102": {
+      "content": "[SEP]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "30526": {
-      "content": "<mask>",
-      "lstrip": true,
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
-  "cls_token": "<s>",
+  "cls_token": "[CLS]",
   "do_basic_tokenize": true,
   "do_lower_case": true,
-  "eos_token": "</s>",
-  "mask_token": "<mask>",
-  "model_max_length": 512,
+  "mask_token": "[MASK]",
+  "model_max_length": 128,
   "never_split": null,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "MPNetTokenizer",
+  "tokenizer_class": "BertTokenizer",
   "unk_token": "[UNK]"
 }

vocab.txt

@@ -1,7 +1,3 @@
-<s>
-<pad>
-</s>
-<unk>
 [PAD]
 [unused0]
 [unused1]
@@ -30524,4 +30520,3 @@ necessitated
 ##：
 ##？
 ##～
-<mask>