update model

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 5,
+  "eos_token_id": 4,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 256,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": null,
+  "position_biased_input": true,
+  "relative_attention": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.1",
+  "type_vocab_size": 0,
+  "vocab_size": 80000
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b39953d7570d3b0611c3b17ae2e487f97112e0e9376f57f2453312a59e13876
+size 419387373

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d13781724629b8e0fda5545b9ddd32267f634323cd62ac71876f2ceb9dc8a308
+size 1600285

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": true,
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "name_or_path": "hieule/vie-spm-uncased-80k",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": "/home/viet/.cache/huggingface/transformers/9e82dafff602fe591d6dcb10539a07b733b6ba6d1da0ad0413ea069c6cf96a45.33ea9968fa7ac107543fcd0be7da0a5b475c882cb5390b8ce2fc4a0c34fe4d6b",
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]"
+}

train_result.log

@@ -0,0 +1,107 @@
+Before load dataset, RAM used: 0.38 GB | Avaiable: 51.32 GB | Left: 50.94 GB
+Dataset({
+    features: ['text'],
+    num_rows: 7625355
+})
+After load dataset, RAM used: 1.78 GB | Avaiable: 50.99 GB | Left: 49.22 GB
+After Prepare Dataloader, RAM used: 2.30 GB | Avaiable: 46.91 GB | Left: 44.61 GB
+After epoch 1, RAM used: 3.57 GB | Avaiable: 51.76 GB | Left: 48.19 GB
+
+ >>> Epoch 1: Perplexity: 21.590424136414132  Loss: 2.6214573416621008
+Loss improved inf -> 2.6214573416621008
+Saved training checkpoint
+After epoch 2, RAM used: 3.53 GB | Avaiable: 57.51 GB | Left: 53.98 GB
+
+ >>> Epoch 2: Perplexity: 14.599685600614704  Loss: 2.4090022345374362
+Loss improved 2.6214573416621008 -> 2.4090022345374362
+Saved training checkpoint
+After epoch 3, RAM used: 3.57 GB | Avaiable: 57.47 GB | Left: 53.89 GB
+
+ >>> Epoch 3: Perplexity: 11.945347969346386  Loss: 2.3160537090725533
+Loss improved 2.4090022345374362 -> 2.3160537090725533
+Saved training checkpoint
+After epoch 4, RAM used: 3.58 GB | Avaiable: 57.47 GB | Left: 53.90 GB
+
+ >>> Epoch 4: Perplexity: 11.61526404278414  Loss: 2.2554892432450773
+Loss improved 2.3160537090725533 -> 2.2554892432450773
+Saved training checkpoint
+After epoch 5, RAM used: 3.58 GB | Avaiable: 52.30 GB | Left: 48.72 GB
+
+ >>> Epoch 5: Perplexity: 10.940166697585614  Loss: 2.211717551305325
+Loss improved 2.2554892432450773 -> 2.211717551305325
+Saved training checkpoint
+After epoch 6, RAM used: 3.58 GB | Avaiable: 57.48 GB | Left: 53.90 GB
+
+ >>> Epoch 6: Perplexity: 9.703375475135896  Loss: 2.1727879395655756
+Loss improved 2.211717551305325 -> 2.1727879395655756
+Saved training checkpoint
+After epoch 7, RAM used: 3.58 GB | Avaiable: 57.46 GB | Left: 53.89 GB
+
+ >>> Epoch 7: Perplexity: 9.611460056753156  Loss: 2.139744746335077
+Loss improved 2.1727879395655756 -> 2.139744746335077
+Saved training checkpoint
+After epoch 8, RAM used: 3.58 GB | Avaiable: 57.45 GB | Left: 53.88 GB
+
+ >>> Epoch 8: Perplexity: 9.253905521907615  Loss: 2.112046389352141
+Loss improved 2.139744746335077 -> 2.112046389352141
+Saved training checkpoint
+After epoch 9, RAM used: 3.58 GB | Avaiable: 57.45 GB | Left: 53.87 GB
+
+ >>> Epoch 9: Perplexity: 8.987782076156853  Loss: 2.088010660353402
+Loss improved 2.112046389352141 -> 2.088010660353402
+Saved training checkpoint
+After epoch 10, RAM used: 3.58 GB | Avaiable: 57.46 GB | Left: 53.89 GB
+
+ >>> Epoch 10: Perplexity: 8.989515812427134  Loss: 2.0606724782881045
+Loss improved 2.088010660353402 -> 2.0606724782881045
+Saved training checkpoint
+After epoch 11, RAM used: 3.58 GB | Avaiable: 57.48 GB | Left: 53.91 GB
+
+ >>> Epoch 11: Perplexity: 8.596087054176957  Loss: 2.0449221819049317
+Loss improved 2.0606724782881045 -> 2.0449221819049317
+Saved training checkpoint
+After epoch 12, RAM used: 3.58 GB | Avaiable: 57.43 GB | Left: 53.85 GB
+
+ >>> Epoch 12: Perplexity: 8.133701984722778  Loss: 2.0227558158141825
+Loss improved 2.0449221819049317 -> 2.0227558158141825
+Saved training checkpoint
+After epoch 13, RAM used: 3.58 GB | Avaiable: 57.47 GB | Left: 53.89 GB
+
+ >>> Epoch 13: Perplexity: 8.120926713195095  Loss: 1.9996797158441224
+Loss improved 2.0227558158141825 -> 1.9996797158441224
+Saved training checkpoint
+After epoch 14, RAM used: 3.58 GB | Avaiable: 57.47 GB | Left: 53.90 GB
+
+ >>> Epoch 14: Perplexity: 7.887189857322398  Loss: 1.986047686118474
+Loss improved 1.9996797158441224 -> 1.986047686118474
+Saved training checkpoint
+After epoch 15, RAM used: 3.58 GB | Avaiable: 55.84 GB | Left: 52.26 GB
+
+ >>> Epoch 15: Perplexity: 7.687336654327518  Loss: 1.9684794586581946
+Loss improved 1.986047686118474 -> 1.9684794586581946
+Saved training checkpoint
+After epoch 16, RAM used: 3.58 GB | Avaiable: 57.46 GB | Left: 53.89 GB
+
+ >>> Epoch 16: Perplexity: 7.549170162424603  Loss: 1.9527537560472294
+Loss improved 1.9684794586581946 -> 1.9527537560472294
+Saved training checkpoint
+After epoch 17, RAM used: 3.58 GB | Avaiable: 57.46 GB | Left: 53.88 GB
+
+ >>> Epoch 17: Perplexity: 7.614053464896555  Loss: 1.9367966973686213
+Loss improved 1.9527537560472294 -> 1.9367966973686213
+Saved training checkpoint
+After epoch 18, RAM used: 3.58 GB | Avaiable: 57.47 GB | Left: 53.89 GB
+
+ >>> Epoch 18: Perplexity: 7.137354318384541  Loss: 1.9238058941700293
+Loss improved 1.9367966973686213 -> 1.9238058941700293
+Saved training checkpoint
+After epoch 19, RAM used: 3.58 GB | Avaiable: 51.08 GB | Left: 47.50 GB
+
+ >>> Epoch 19: Perplexity: 7.229089215165024  Loss: 1.9130641898584901
+Loss improved 1.9238058941700293 -> 1.9130641898584901
+Saved training checkpoint
+After epoch 20, RAM used: 3.56 GB | Avaiable: 51.37 GB | Left: 47.80 GB
+
+ >>> Epoch 20: Perplexity: 7.165172113154145  Loss: 1.9069529029687642
+Loss improved 1.9130641898584901 -> 1.9069529029687642
+Saved training checkpoint