Training in progress, step 10

26_10_23_config_test.csv

@@ -6,7 +6,7 @@ run_number,comment,peformed_already,num_train_epochs,max_tokens,temperature,stop
 5,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
 6,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
 7,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
-8,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
+8,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
 9,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
 10,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
 11,classification_of_valuems set True,False,2,100,0.8,False,True,False,False

26_10_23_results_test.csv

@@ -26,4 +26,4 @@ run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnes
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
-0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+0,1528.8057931953265,classification_of_valuems set True,17.268380403518677,0,0.2478655270791332,0.3576253534106945,0.2778231048355419,0.2832970005606974,0.4013383893240502,0.3126046416067341,0.3252384765751978,0.4466121889283891,0.3509020400982625,0.1004556235113589,0.1415746397318885,0.1108019406524126,0.1231052130296416,0.1705496820565623,0.1340090789111166,0.1477750597812323,0.19929804844641,0.1574531105220511,0.25,0.25,0.25

27_10_23_23_results_real_3.csv

@@ -0,0 +1,2 @@
+run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnest,rouge1 low Precision,rouge1 low Recall,rouge1 low F1 Score,rouge1 mid Precision,rouge1 mid Recall,rouge1 mid F1 Score,rouge1 high Precision,rouge1 high Recall,rouge1 high F1 Score,rouge2 low Precision,rouge2 low Recall,rouge2 low F1 Score,rouge2 mid Precision,rouge2 mid Recall,rouge2 mid F1 Score,rouge2 high Precision,rouge2 high Recall,rouge2 high F1 Score,min_cosine_sim_value,max_cosine_sim_value,mean_cosine_sim_value
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

27_10_23_config_test.csv

@@ -0,0 +1,3 @@
+run_number,comment,peformed_already,num_train_epochs,max_tokens,temperature,stop_token,classification_of_valuems,stemming,lemmatization
+1,no variations,False,2,100,0.8,False,False,False,False
+2,classification_of_valuems set True,False,2,100,0.8,False,True,False,False

27_10_23_results_real_3.csv

@@ -0,0 +1,2 @@
+run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnest,rouge1 low Precision,rouge1 low Recall,rouge1 low F1 Score,rouge1 mid Precision,rouge1 mid Recall,rouge1 mid F1 Score,rouge1 high Precision,rouge1 high Recall,rouge1 high F1 Score,rouge2 low Precision,rouge2 low Recall,rouge2 low F1 Score,rouge2 mid Precision,rouge2 mid Recall,rouge2 mid F1 Score,rouge2 high Precision,rouge2 high Recall,rouge2 high F1 Score,min_cosine_sim_value,max_cosine_sim_value,mean_cosine_sim_value
+1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

27_10_23_results_test.csv

@@ -0,0 +1,2 @@
+run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnest,rouge1 low Precision,rouge1 low Recall,rouge1 low F1 Score,rouge1 mid Precision,rouge1 mid Recall,rouge1 mid F1 Score,rouge1 high Precision,rouge1 high Recall,rouge1 high F1 Score,rouge2 low Precision,rouge2 low Recall,rouge2 low F1 Score,rouge2 mid Precision,rouge2 mid Recall,rouge2 mid F1 Score,rouge2 high Precision,rouge2 high Recall,rouge2 high F1 Score,min_cosine_sim_value,max_cosine_sim_value,mean_cosine_sim_value
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

README.md

@@ -1000,6 +1000,18 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_use_double_quant: True
 - bnb_4bit_compute_dtype: bfloat16
 
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - quant_method: bitsandbytes
 - load_in_8bit: False
@@ -1096,5 +1108,6 @@ The following `bitsandbytes` quantization config was used during training:
 - PEFT 0.5.0
 - PEFT 0.5.0
 - PEFT 0.5.0
+- PEFT 0.5.0
 
 - PEFT 0.5.0

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44b0e4641a21400f9ac8b892d5ea86ba119fb8bfd1e2ae0c92a38bef210714a7
+oid sha256:79395823541aa379c4f344f6d050486319a8431f84a8bdd7e1a0d50f6dfc715d
 size 100733709

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc30f8a1f8cbc7e6e739a3bccd24e9771a9a66560696bd18ec550294cffdb65d
-size 261131840
+oid sha256:ea8d177806c82c8311bd985e84d7ca45c95ac8df57ddadbd9bfa3a9503194ada
+size 100690288

config.json

@@ -1,33 +1,33 @@
 {
-  "_name_or_path": "tiiuae/falcon-7b",
-  "alibi": false,
+  "_name_or_path": "tiiuae/falcon-rw-1b",
+  "alibi": true,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
     "FalconForCausalLM"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "tiiuae/falcon-7b--configuration_falcon.FalconConfig",
-    "AutoModel": "tiiuae/falcon-7b--modeling_falcon.FalconModel",
-    "AutoModelForCausalLM": "tiiuae/falcon-7b--modeling_falcon.FalconForCausalLM",
-    "AutoModelForQuestionAnswering": "tiiuae/falcon-7b--modeling_falcon.FalconForQuestionAnswering",
-    "AutoModelForSequenceClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForSequenceClassification",
-    "AutoModelForTokenClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForTokenClassification"
+    "AutoConfig": "tiiuae/falcon-rw-1b--configuration_falcon.FalconConfig",
+    "AutoModel": "tiiuae/falcon-rw-1b--modeling_falcon.FalconModel",
+    "AutoModelForCausalLM": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForCausalLM",
+    "AutoModelForQuestionAnswering": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForQuestionAnswering",
+    "AutoModelForSequenceClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForSequenceClassification",
+    "AutoModelForTokenClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForTokenClassification"
   },
-  "bias": false,
-  "bos_token_id": 11,
-  "eos_token_id": 11,
+  "bias": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
   "hidden_dropout": 0.0,
-  "hidden_size": 4544,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "falcon",
-  "multi_query": true,
+  "multi_query": false,
   "new_decoder_architecture": false,
-  "num_attention_heads": 71,
-  "num_hidden_layers": 32,
-  "num_kv_heads": 71,
-  "parallel_attn": true,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 24,
+  "num_kv_heads": 32,
+  "parallel_attn": false,
   "quantization_config": {
     "bnb_4bit_compute_dtype": "bfloat16",
     "bnb_4bit_quant_type": "nf4",
@@ -43,5 +43,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.34.1",
   "use_cache": false,
-  "vocab_size": 65024
+  "vocab_size": 50304
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:748da944795546b277c6e7e61d431e15d33d32dc5c1ea3807fa94ebde545837d
+oid sha256:2be2e6c88b186be5fb357782826030044d7a59da0f0613012d4cb7645606a2fe
 size 4283