hpc model

README.md

@@ -1,3 +1,100 @@
 ---
+datasets:
+- BramVanroy/hebban-reviews
+language:
+- nl
 license: mit
+metrics:
+- accuracy
+- f1
+- precision
+- qwk
+- recall
+model-index:
+- name: bert-base-multilingual-cased-hebban-reviews5
+  results:
+  - dataset:
+      config: filtered_rating
+      name: BramVanroy/hebban-reviews - filtered_rating - 2.0.0
+      revision: 2.0.0
+      split: test
+      type: BramVanroy/hebban-reviews
+    metrics:
+    - name: Test accuracy
+      type: accuracy
+      value: 0.5898668639053254
+    - name: Test f1
+      type: f1
+      value: 0.5899204480029937
+    - name: Test precision
+      type: precision
+      value: 0.5971431895675179
+    - name: Test qwk
+      type: qwk
+      value: 0.7050840079198698
+    - name: Test recall
+      type: recall
+      value: 0.5898668639053254
+    task:
+      name: sentiment analysis
+      type: text-classification
+tags:
+- sentiment-analysis
+- dutch
+- text
+widget:
+- text: Wauw, wat een leuk boek! Ik heb me er er goed mee vermaakt.
+- text: Nee, deze vond ik niet goed. De auteur doet zijn best om je als lezer mee
+    te trekken in het verhaal maar mij overtuigt het alleszins niet.
+- text: Ik vind het niet slecht maar de schrijfstijl trekt me ook niet echt aan. Het
+    wordt een beetje saai vanaf het vijfde hoofdstuk
 ---
+
+# bert-base-multilingual-cased-hebban-reviews
+
+# Dataset
+- dataset_name: BramVanroy/hebban-reviews
+- dataset_config: filtered_rating
+- dataset_revision: 2.0.0
+- labelcolumn: review_rating0
+- textcolumn: review_text_without_quotes
+
+# Training
+- optim: adamw_hf
+- learning_rate: 5e-05
+- per_device_train_batch_size: 64
+- per_device_eval_batch_size: 64
+- gradient_accumulation_steps: 1
+- max_steps: 5001
+- save_steps: 500
+- metric_for_best_model: qwk
+
+# Best checkedpoint based on validation
+- best_metric: 0.697825193570947
+- best_model_checkpoint: trained/hebban-reviews5/bert-base-multilingual-cased/checkpoint-4500
+
+# Test results of best checkpoint
+- accuracy: 0.5898668639053254
+- f1: 0.5899204480029937
+- precision: 0.5971431895675179
+- qwk: 0.7050840079198698
+- recall: 0.5898668639053254
+
+## Confusion matric
+
+![cfm](fig/test_confusion_matrix.png)
+
+## Normalized confusion matrix
+
+![norm cfm](fig/test_confusion_matrix_norm.png)
+
+# Environment
+- cuda_capabilities: 8.0; 8.0
+- cuda_device_count: 2
+- cuda_devices: NVIDIA A100-SXM4-80GB; NVIDIA A100-SXM4-80GB
+- finetuner_commit: 8159b4c1d5e66b36f68dd263299927ffb8670ebd
+- platform: Linux-4.18.0-305.49.1.el8_4.x86_64-x86_64-with-glibc2.28
+- python_version: 3.9.5
+- toch_version: 1.10.0
+- transformers_version: 4.21.0
+

all_results.json

@@ -0,0 +1,12 @@
+{
+    "accuracy": 0.5898668639053254,
+    "epoch": 4.38,
+    "f1": 0.5899204480029937,
+    "precision": 0.5971431895675179,
+    "qwk": 0.7050840079198698,
+    "recall": 0.5898668639053254,
+    "train_loss": 0.8685172849525287,
+    "train_runtime": 2653.5735,
+    "train_samples_per_second": 241.232,
+    "train_steps_per_second": 1.885
+}

args.json

@@ -0,0 +1,136 @@
+{
+    "_n_gpu": 1,
+    "adafactor": false,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.999,
+    "adam_epsilon": 1e-08,
+    "auto_find_batch_size": false,
+    "bf16": false,
+    "bf16_full_eval": false,
+    "calculate_qwk": true,
+    "data_seed": 42,
+    "dataloader_drop_last": false,
+    "dataloader_num_workers": 0,
+    "dataloader_pin_memory": true,
+    "dataset_config": "filtered_rating",
+    "dataset_name": "BramVanroy/hebban-reviews",
+    "dataset_revision": "2.0.0",
+    "ddp_bucket_cap_mb": null,
+    "ddp_find_unused_parameters": null,
+    "debug": [],
+    "deepspeed": null,
+    "disable_tqdm": false,
+    "do_early_stopping": false,
+    "do_eval": true,
+    "do_optimize": false,
+    "do_predict": true,
+    "do_train": true,
+    "early_stopping_patience": 1,
+    "early_stopping_threshold": 0.0,
+    "eval_accumulation_steps": null,
+    "eval_delay": 0,
+    "eval_steps": 500,
+    "evaluation_strategy": "steps",
+    "fp16": true,
+    "fp16_backend": "auto",
+    "fp16_full_eval": false,
+    "fp16_opt_level": "O1",
+    "fsdp": [],
+    "fsdp_min_num_params": 0,
+    "fsdp_transformer_layer_cls_to_wrap": null,
+    "full_determinism": false,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": false,
+    "greater_is_better": true,
+    "group_by_length": false,
+    "half_precision_backend": "cuda_amp",
+    "hub_model_id": null,
+    "hub_private_repo": false,
+    "hub_strategy": "every_save",
+    "hub_token": null,
+    "ignore_data_skip": false,
+    "include_inputs_for_metrics": false,
+    "jit_mode_eval": false,
+    "label_names": null,
+    "label_smoothing_factor": 0.0,
+    "labelcolumn": "review_rating0",
+    "labelnames": [
+        "1",
+        "2",
+        "3",
+        "4",
+        "5"
+    ],
+    "learning_rate": 5e-05,
+    "length_column_name": "length",
+    "load_best_model_at_end": true,
+    "local_rank": 0,
+    "log_level": -1,
+    "log_level_replica": -1,
+    "log_on_each_node": true,
+    "logging_dir": "trained/hebban-reviews5/bert-base-multilingual-cased/runs/Jul29_00-34-45_node3907.accelgor.os",
+    "logging_first_step": false,
+    "logging_nan_inf_filter": true,
+    "logging_steps": 500,
+    "logging_strategy": "steps",
+    "lr_scheduler_type": "linear",
+    "max_grad_norm": 1.0,
+    "max_seq_length": null,
+    "max_steps": 5001,
+    "max_test_samples": null,
+    "max_train_samples": null,
+    "max_validation_samples": null,
+    "metric_for_best_model": "qwk",
+    "model_name_or_path": "bert-base-multilingual-cased",
+    "model_revision": "main",
+    "mp_parameters": "",
+    "n_trials": 8,
+    "no_cuda": false,
+    "num_train_epochs": 3.0,
+    "optim": "adamw_hf",
+    "output_dir": "trained/hebban-reviews5/bert-base-multilingual-cased",
+    "overwrite_cache": false,
+    "overwrite_output_dir": true,
+    "past_index": -1,
+    "per_device_eval_batch_size": 64,
+    "per_device_train_batch_size": 64,
+    "per_gpu_eval_batch_size": null,
+    "per_gpu_train_batch_size": null,
+    "prediction_loss_only": false,
+    "push_to_hub": false,
+    "push_to_hub_model_id": null,
+    "push_to_hub_organization": null,
+    "push_to_hub_token": null,
+    "ray_scope": "all",
+    "remove_unused_columns": true,
+    "report_to": [
+        "tensorboard"
+    ],
+    "resume_from_checkpoint": null,
+    "run_name": "trained/hebban-reviews5/bert-base-multilingual-cased",
+    "save_on_each_node": false,
+    "save_steps": 500,
+    "save_strategy": "steps",
+    "save_total_limit": null,
+    "scheduler_type": null,
+    "seed": 42,
+    "sharded_ddp": [],
+    "skip_memory_metrics": true,
+    "split_seed": 42,
+    "testsplit_name": "test",
+    "textcolumn": "review_text_without_quotes",
+    "tf32": null,
+    "torchdynamo": null,
+    "tpu_metrics_debug": false,
+    "tpu_num_cores": null,
+    "trainsplit_name": "train",
+    "use_class_weights": true,
+    "use_ipex": false,
+    "use_legacy_prediction_loop": false,
+    "validation_size": 0.1,
+    "validationsplit_name": "validation",
+    "warmup_ratio": 0.0,
+    "warmup_steps": 0,
+    "weight_decay": 0.0,
+    "xpu_backend": null
+}

config.json

@@ -0,0 +1,45 @@
+{
+  "_name_or_path": "bert-base-multilingual-cased",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "1",
+    "1": "2",
+    "2": "3",
+    "3": "4",
+    "4": "5"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "1": 0,
+    "2": 1,
+    "3": 2,
+    "4": 3,
+    "5": 4
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 119547
+}

env.json

@@ -0,0 +1,10 @@
+{
+    "cuda_capabilities": "8.0; 8.0",
+    "cuda_device_count": 2,
+    "cuda_devices": "NVIDIA A100-SXM4-80GB; NVIDIA A100-SXM4-80GB",
+    "finetuner_commit": "8159b4c1d5e66b36f68dd263299927ffb8670ebd",
+    "platform": "Linux-4.18.0-305.49.1.el8_4.x86_64-x86_64-with-glibc2.28",
+    "python_version": "3.9.5",
+    "toch_version": "1.10.0",
+    "transformers_version": "4.21.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8c2d40ee3db8a5806492e53ca9510899d2d5f16169b722ad858b7e134a7f503
+size 711500141

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

test_predictions.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a20b01868f6a528dbcf295b8388d9c07ad1379bddbfdce3fb931c28260036f4
+size 65128043

test_results.json

@@ -0,0 +1,7 @@
+{
+    "accuracy": 0.5898668639053254,
+    "f1": 0.5899204480029937,
+    "precision": 0.5971431895675179,
+    "qwk": 0.7050840079198698,
+    "recall": 0.5898668639053254
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "bert-base-multilingual-cased",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 4.38,
+    "train_loss": 0.8685172849525287,
+    "train_runtime": 2653.5735,
+    "train_samples_per_second": 241.232,
+    "train_steps_per_second": 1.885
+}

trainer_state.json

@@ -0,0 +1,215 @@
+{
+  "best_metric": 0.697825193570947,
+  "best_model_checkpoint": "trained/hebban-reviews5/bert-base-multilingual-cased/checkpoint-4500",
+  "epoch": 4.382997370727432,
+  "global_step": 5001,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.502099580083983e-05,
+      "loss": 1.3159,
+      "step": 500
+    },
+    {
+      "epoch": 0.44,
+      "eval_accuracy": 0.4912475345167653,
+      "eval_f1": 0.48070042281702435,
+      "eval_loss": 1.1863828897476196,
+      "eval_precision": 0.5054226630223979,
+      "eval_qwk": 0.6104106009981652,
+      "eval_recall": 0.4912475345167653,
+      "eval_runtime": 23.5101,
+      "eval_samples_per_second": 690.086,
+      "eval_steps_per_second": 5.402,
+      "step": 500
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 4.003199360127974e-05,
+      "loss": 1.092,
+      "step": 1000
+    },
+    {
+      "epoch": 0.88,
+      "eval_accuracy": 0.5488165680473372,
+      "eval_f1": 0.5517553489817303,
+      "eval_loss": 1.0387530326843262,
+      "eval_precision": 0.5678859988610889,
+      "eval_qwk": 0.6346615351470672,
+      "eval_recall": 0.5488165680473372,
+      "eval_runtime": 23.5201,
+      "eval_samples_per_second": 689.794,
+      "eval_steps_per_second": 5.4,
+      "step": 1000
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 3.5032993401319734e-05,
+      "loss": 0.9709,
+      "step": 1500
+    },
+    {
+      "epoch": 1.31,
+      "eval_accuracy": 0.5436390532544378,
+      "eval_f1": 0.5408623179767048,
+      "eval_loss": 1.0117485523223877,
+      "eval_precision": 0.5642348913139151,
+      "eval_qwk": 0.6678503252600388,
+      "eval_recall": 0.5436390532544378,
+      "eval_runtime": 23.6496,
+      "eval_samples_per_second": 686.015,
+      "eval_steps_per_second": 5.37,
+      "step": 1500
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 3.0033993201359727e-05,
+      "loss": 0.9404,
+      "step": 2000
+    },
+    {
+      "epoch": 1.75,
+      "eval_accuracy": 0.5279215976331361,
+      "eval_f1": 0.5218328433667149,
+      "eval_loss": 1.0087615251541138,
+      "eval_precision": 0.5543969942392576,
+      "eval_qwk": 0.6748096376745845,
+      "eval_recall": 0.5279215976331361,
+      "eval_runtime": 23.476,
+      "eval_samples_per_second": 691.088,
+      "eval_steps_per_second": 5.41,
+      "step": 2000
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 2.503499300139972e-05,
+      "loss": 0.8657,
+      "step": 2500
+    },
+    {
+      "epoch": 2.19,
+      "eval_accuracy": 0.5708826429980276,
+      "eval_f1": 0.5692427852436814,
+      "eval_loss": 1.0947091579437256,
+      "eval_precision": 0.5827881052454773,
+      "eval_qwk": 0.6863988830193634,
+      "eval_recall": 0.5708826429980276,
+      "eval_runtime": 23.3841,
+      "eval_samples_per_second": 693.805,
+      "eval_steps_per_second": 5.431,
+      "step": 2500
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 2.0035992801439712e-05,
+      "loss": 0.8004,
+      "step": 3000
+    },
+    {
+      "epoch": 2.63,
+      "eval_accuracy": 0.5539940828402367,
+      "eval_f1": 0.5591418616361084,
+      "eval_loss": 1.0360697507858276,
+      "eval_precision": 0.5772334772150153,
+      "eval_qwk": 0.6712182177231302,
+      "eval_recall": 0.5539940828402367,
+      "eval_runtime": 23.5321,
+      "eval_samples_per_second": 689.441,
+      "eval_steps_per_second": 5.397,
+      "step": 3000
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 1.5046990601879624e-05,
+      "loss": 0.7687,
+      "step": 3500
+    },
+    {
+      "epoch": 3.07,
+      "eval_accuracy": 0.5808678500986193,
+      "eval_f1": 0.5832607549463982,
+      "eval_loss": 1.2274583578109741,
+      "eval_precision": 0.5914789641012546,
+      "eval_qwk": 0.689381177306146,
+      "eval_recall": 0.5808678500986193,
+      "eval_runtime": 23.7912,
+      "eval_samples_per_second": 681.932,
+      "eval_steps_per_second": 5.338,
+      "step": 3500
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 1.0047990401919617e-05,
+      "loss": 0.6773,
+      "step": 4000
+    },
+    {
+      "epoch": 3.51,
+      "eval_accuracy": 0.5830251479289941,
+      "eval_f1": 0.5826927994622666,
+      "eval_loss": 1.222939133644104,
+      "eval_precision": 0.5930910696407776,
+      "eval_qwk": 0.6959801434105679,
+      "eval_recall": 0.5830251479289941,
+      "eval_runtime": 23.6989,
+      "eval_samples_per_second": 684.589,
+      "eval_steps_per_second": 5.359,
+      "step": 4000
+    },
+    {
+      "epoch": 3.94,
+      "learning_rate": 5.048990201959608e-06,
+      "loss": 0.6584,
+      "step": 4500
+    },
+    {
+      "epoch": 3.94,
+      "eval_accuracy": 0.5886341222879684,
+      "eval_f1": 0.5886427250282493,
+      "eval_loss": 1.281541347503662,
+      "eval_precision": 0.5969768120377853,
+      "eval_qwk": 0.697825193570947,
+      "eval_recall": 0.5886341222879684,
+      "eval_runtime": 23.7004,
+      "eval_samples_per_second": 684.544,
+      "eval_steps_per_second": 5.359,
+      "step": 4500
+    },
+    {
+      "epoch": 4.38,
+      "learning_rate": 4.999000199960008e-08,
+      "loss": 0.5961,
+      "step": 5000
+    },
+    {
+      "epoch": 4.38,
+      "eval_accuracy": 0.5894354043392505,
+      "eval_f1": 0.5909264551740898,
+      "eval_loss": 1.359961748123169,
+      "eval_precision": 0.5997451050742093,
+      "eval_qwk": 0.6969542849451541,
+      "eval_recall": 0.5894354043392505,
+      "eval_runtime": 23.5424,
+      "eval_samples_per_second": 689.138,
+      "eval_steps_per_second": 5.395,
+      "step": 5000
+    },
+    {
+      "epoch": 4.38,
+      "step": 5001,
+      "total_flos": 1.6839139993111757e+17,
+      "train_loss": 0.8685172849525287,
+      "train_runtime": 2653.5735,
+      "train_samples_per_second": 241.232,
+      "train_steps_per_second": 1.885
+    }
+  ],
+  "max_steps": 5001,
+  "num_train_epochs": 5,
+  "total_flos": 1.6839139993111757e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e298551f089369377fb5d6d386f38e031de6fb9fbac68db00070b86d1cae8f97
+size 3439