Upload 9 files

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "cl-tohoku/bert-base-japanese-whole-word-masking",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "positive",
+    "1": "negative",
+    "2": "neutral"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 1,
+    "neutral": 2,
+    "positive": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 32000
+}

finetuning_wrime_01_base.py

@@ -0,0 +1,172 @@
+# %%
+import torch
+# GPUが使用可能か判断
+if torch.cuda.is_available():
+    print('gpu is available')
+else:
+    raise Exception('gpu is NOT available')
+
+# %%
+from datasets import load_dataset, DatasetDict
+from transformers import AutoTokenizer
+from transformers import AutoModelForSequenceClassification
+from transformers import TrainingArguments
+from transformers import Trainer
+from sklearn.metrics import accuracy_score, f1_score
+import numpy as np
+import pandas as pd
+import torch
+import random
+
+# %%
+from pprint import pprint
+from datasets import load_dataset
+
+# Hugging Face Hub上のllm-book/wrime-sentimentのリポジトリから
+# データを読み込む
+train_dataset = load_dataset("llm-book/wrime-sentiment", split="train", remove_neutral=False)
+valid_dataset = load_dataset("llm-book/wrime-sentiment", split="validation", remove_neutral=False)
+# pprintで見やすく表示する
+pprint(train_dataset)
+pprint(valid_dataset)
+
+# %%
+# トークナイザのロード
+model_name = "cl-tohoku/bert-base-japanese-whole-word-masking"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+# %%
+# トークナイズ処理
+def preprocess_text(batch):
+    encoded_batch = tokenizer(batch['sentence'], max_length=512)
+    encoded_batch['labels'] = batch['label']
+    return encoded_batch
+
+encoded_train_dataset = train_dataset.map(
+    preprocess_text,
+    remove_columns=train_dataset.column_names,
+)
+encoded_valid_dataset = valid_dataset.map(
+    preprocess_text,
+    remove_columns=valid_dataset.column_names,
+)
+
+# ミニバッチ構築
+from transformers import DataCollatorWithPadding
+
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+# %%
+# モデルの準備
+from transformers import AutoModelForSequenceClassification
+
+class_label = train_dataset.features["label"]
+label2id = {label: id for id, label in enumerate(class_label.names)}
+id2label = {id: label for id, label in enumerate(class_label.names)}
+model = AutoModelForSequenceClassification.from_pretrained(
+    model_name,
+    num_labels=class_label.num_classes,
+    label2id=label2id,  # ラベル名からIDへの対応を指定
+    id2label=id2label,  # IDからラベル名への対応を指定
+)
+print(type(model).__name__)
+
+# %%
+# 訓練の準備
+from transformers import TrainingArguments
+# 保存ディレクトリ
+save_dir = f'bert-finetuned-wrime-base'
+
+training_args = TrainingArguments(
+    output_dir=save_dir,  # 結果の保存フォルダ
+    per_device_train_batch_size=32,  # 訓練時のバッチサイズ
+    per_device_eval_batch_size=32,  # 評価時のバッチサイズ
+    learning_rate=2e-5,  # 学習率
+    lr_scheduler_type="constant",  # 学習率スケジューラの種類
+    warmup_ratio=0.1,  # 学習率のウォームアップの長さを指定
+    num_train_epochs=100,  # エポック数
+    save_strategy="epoch",  # チェックポイントの保存タイミング
+    logging_strategy="epoch",  # ロギングのタイミング
+    evaluation_strategy="epoch",  # 検証セットによる評価のタイミング
+    load_best_model_at_end=True,  # 訓練後に開発セットで最良のモデルをロード
+    metric_for_best_model="accuracy",  # 最良のモデルを決定する評価指標
+    fp16=True,  # 自動混合精度演算の有効化
+)
+
+# %%
+# メトリクスの定義
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = pred.predictions.argmax(-1)
+    f1 = f1_score(labels, preds, average="weighted")
+    acc = accuracy_score(labels, preds)
+    return {"accuracy": acc, "f1": f1}
+
+# %%
+# 訓練の実行
+from transformers import Trainer
+from transformers import EarlyStoppingCallback
+
+trainer = Trainer(
+    model=model,
+    train_dataset=encoded_train_dataset,
+    eval_dataset=encoded_valid_dataset,
+    data_collator=data_collator,
+    args=training_args,
+    compute_metrics=compute_metrics,
+    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
+)
+trainer.train()
+
+# %%
+# モデルの保存
+trainer.save_model(save_dir)
+tokenizer.save_pretrained(save_dir)
+# 履歴の保存
+history_df = pd.DataFrame(trainer.state.log_history)
+history_df.to_csv('base_line/wrime_baseline_history.csv')
+
+# %%
+import matplotlib.pyplot as plt
+
+def show_graph(df, suptitle, output='output.png'):
+    suptitle_size = 23
+    graph_title_size = 20
+    legend_size = 18
+    ticks_size = 13
+    # 学習曲線
+    fig = plt.figure(figsize=(20, 5))
+    plt.suptitle(suptitle, fontsize=suptitle_size)
+    # Train Loss
+    plt.subplot(131)
+    plt.title('Train Loss', fontsize=graph_title_size)
+    plt.plot(df['loss'].dropna(), label='train')
+    plt.legend(fontsize=legend_size)
+    plt.yticks(fontsize=ticks_size)
+    # Validation Loss
+    plt.subplot(132)
+    # reg_str = f'$y={round(regression.coef_[0],5)}*x+{round(regression.intercept_,3)}$'
+    plt.title(f'Val Loss', fontsize=graph_title_size)
+    y = df['eval_loss'].dropna().values
+    x = np.arange(len(y)).reshape(-1, 1)
+    # pred = regression.coef_ * x.ravel() + regression.intercept_  # 線形回帰直線
+    plt.plot(y, color='tab:orange', label='val')
+    # plt.plot(pred, color='green', label='pred')
+    plt.legend(fontsize=legend_size)
+    # plt.xlabel(reg_str, fontsize=ticks_size)
+    plt.yticks(fontsize=ticks_size)
+    # Accuracy/F1
+    plt.subplot(133)
+    plt.title('eval Accuracy/F1', fontsize=graph_title_size)
+    plt.plot(df['eval_accuracy'].dropna(), label='accuracy')
+    plt.plot(df['eval_f1'].dropna(), label='F1')
+    plt.legend(fontsize=legend_size)
+    plt.yticks(fontsize=ticks_size)
+    plt.tight_layout()
+    # plt.show()
+    plt.savefig(output)
+
+# %%
+# 結果を表示
+suptitle = 'batch:32, lr:2e-5, type:constant'
+show_graph(history_df, suptitle, 'base_line/wrime_baseline_output.png')

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc31eef7269ace955c7c243db26a97edcdf58e42d43d477fd124ff3a9354ac1
+size 442545135

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "do_subword_tokenize": true,
+  "do_word_tokenize": true,
+  "jumanpp_kwargs": null,
+  "mask_token": "[MASK]",
+  "mecab_kwargs": null,
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "subword_tokenizer_type": "wordpiece",
+  "sudachi_kwargs": null,
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "unk_token": "[UNK]",
+  "word_tokenizer_type": "mecab"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9015a0155086e38ddabd69ddde783cf2e6da0f931df8984a6996b23e63cf1a0a
+size 4015

wrime_baseline_history.csv

@@ -0,0 +1,12 @@
+,loss,learning_rate,epoch,step,eval_loss,eval_accuracy,eval_f1,eval_runtime,eval_samples_per_second,eval_steps_per_second,train_runtime,train_samples_per_second,train_steps_per_second,total_flos,train_loss
+0,0.7499,2e-05,1.0,938,,,,,,,,,,,
+1,,,1.0,938,0.654792308807373,0.712,0.712652463040488,1.2726,1964.409,62.075,,,,,
+2,0.5464,2e-05,2.0,1876,,,,,,,,,,,
+3,,,2.0,1876,0.699586033821106,0.7128,0.7073848402995836,1.2761,1959.062,61.906,,,,,
+4,0.3782,2e-05,3.0,2814,,,,,,,,,,,
+5,,,3.0,2814,0.7840703129768372,0.7028,0.7011891803629126,1.2756,1959.882,61.932,,,,,
+6,0.2444,2e-05,4.0,3752,,,,,,,,,,,
+7,,,4.0,3752,0.9180415868759155,0.7108,0.7088532209184721,1.2745,1961.551,61.985,,,,,
+8,0.1664,2e-05,5.0,4690,,,,,,,,,,,
+9,,,5.0,4690,1.0885692834854126,0.7128,0.7127755757268002,1.2786,1955.302,61.788,,,,,
+10,,,5.0,4690,,,,,,,328.8299,9123.257,285.254,6502067001188640.0,0.41708059737931436