add

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+datasets/mix_dataset_2022_08_30.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,21 @@
+import gradio as gr
+from extraction import paragraph_extract
+
+def predict(paragraphs, positions):
+    paragraphs = [paragraphs]
+    positions = [positions]
+    return extractor(paragraphs, positions)[0]
+
+extractor = paragraph_extract().extract
+
+example_paragraph = 'The W/Zr/HfO2 /TiN structure was fabricated following the scheme shown in the inset of Fig. 1(a). A 5-nm-thick HfO2 layer was deposited on a TiN substrate by an atomic layer deposition system. After HfO2 film deposition, thermal annealing was performed under NH3 at 700∘C in order to achieve optimum concentration of oxygen vacancies [10]. Then, the 3-nm-thick Zr top electrode and a 50-nm-thick W capping layer were deposited by RF magnetron sputtering system. The size of the upper electrode was 10×10 μm2 . The electrical measurements were performed by an Agilent B1500A semiconductor device analyzer, equipped with two pulse generator modules WGFMU (Waveform Generator and Fast Measurement Unit). The coaxial cables with a 50-Ω resistance and less than 10 cm in length were used to reduce the parasitic effects.'
+example_position = 4
+
+demo = gr.Interface(fn=predict, inputs=[gr.inputs.Textbox(lines=3, label="Paragraphs", placeholder='Text Here...'), 
+                                        gr.inputs.Number(label="Positions")], 
+                    outputs="text", 
+                    title="ReRAM Paragraph Classification", allow_flagging=False,
+                    examples=[[example_paragraph, example_position]],
+                    )
+
+demo.launch(share=True)

configs/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dfa79fd27e2577a6dfedb39d439cee4e4d823789fe844127a930f179b04592d
+size 442225641

configs/config.yaml

@@ -0,0 +1,30 @@
+train:
+  epochs: 20
+  batch_size: 16
+  lr: 1.0E-5
+  dropout: 0.1
+  data_cut: null
+  early_stop_count: 5
+
+wandb:
+  wandb_log: True
+  wandb_project: 'paragraph_classification'
+  wandb_group: 'model_test'
+  wandb_memo: 'scibert'
+  wandb_name: 'scibert'
+
+model:
+  model_name: 'allenai/scibert_scivocab_cased'
+  data_file: './datasets/mix_dataset_2022_08_30.csv'
+  max_length: 512
+  random_state: 1000
+  task_type: 'scalar'
+  freeze_layers: null
+  num_classifier: 1
+  num_pos_emb_layer: 1
+  sentence_piece: False
+  bertsum: False
+
+extract:
+  selected_model: 'configs/best_model.pt'
+  batch_size: 16

datasets/mix_dataset_2022_08_30.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd4fe999f4289ed8840dd9600958da1ad8c11caa29fca8eb6ce89a6e42df391a
+size 13741487

example.ipynb

@@ -0,0 +1,122 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/rnwnsgud1234/0.Files/anaconda3/envs/nlp/lib/python3.9/site-packages/gradio/inputs.py:26: UserWarning: Usage of gradio.inputs is deprecated, and will not be supported in the future, please import your component from gradio.components\n",
+      "  warnings.warn(\n",
+      "/home/rnwnsgud1234/0.Files/anaconda3/envs/nlp/lib/python3.9/site-packages/gradio/deprecation.py:40: UserWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  warnings.warn(value)\n",
+      "/home/rnwnsgud1234/0.Files/anaconda3/envs/nlp/lib/python3.9/site-packages/gradio/deprecation.py:40: UserWarning: `numeric` parameter is deprecated, and it has no effect\n",
+      "  warnings.warn(value)\n",
+      "/home/rnwnsgud1234/0.Files/anaconda3/envs/nlp/lib/python3.9/site-packages/gradio/inputs.py:58: UserWarning: Usage of gradio.inputs is deprecated, and will not be supported in the future, please import your component from gradio.components\n",
+      "  warnings.warn(\n",
+      "/home/rnwnsgud1234/0.Files/anaconda3/envs/nlp/lib/python3.9/site-packages/gradio/interface.py:359: UserWarning: The `allow_flagging` parameter in `Interface` nowtakes a string value ('auto', 'manual', or 'never'), not a boolean. Setting parameter to: 'never'.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7862\n",
+      "Running on public URL: https://bd6b9acba15cf888.gradio.app\n",
+      "\n",
+      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades (NEW!), check out Spaces: https://huggingface.co/spaces\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"https://bd6b9acba15cf888.gradio.app\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1/1 [00:01<00:00,  1.37s/it]\n",
+      "100%|██████████| 1/1 [00:00<00:00, 67.94it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "from extraction import paragraph_extract\n",
+    "\n",
+    "def predict(paragraphs, positions):\n",
+    "    paragraphs = [paragraphs]\n",
+    "    positions = [positions]\n",
+    "    return extractor(paragraphs, positions)[0]\n",
+    "\n",
+    "extractor = paragraph_extract().extract\n",
+    "\n",
+    "example_paragraph = 'The W/Zr/HfO2 /TiN structure was fabricated following the scheme shown in the inset of Fig. 1(a). A 5-nm-thick HfO2 layer was deposited on a TiN substrate by an atomic layer deposition system. After HfO2 film deposition, thermal annealing was performed under NH3 at 700∘C in order to achieve optimum concentration of oxygen vacancies [10]. Then, the 3-nm-thick Zr top electrode and a 50-nm-thick W capping layer were deposited by RF magnetron sputtering system. The size of the upper electrode was 10×10 μm2 . The electrical measurements were performed by an Agilent B1500A semiconductor device analyzer, equipped with two pulse generator modules WGFMU (Waveform Generator and Fast Measurement Unit). The coaxial cables with a 50-Ω resistance and less than 10 cm in length were used to reduce the parasitic effects.'\n",
+    "example_position = 4\n",
+    "\n",
+    "demo = gr.Interface(fn=predict, inputs=[gr.inputs.Textbox(lines=3, label=\"Paragraphs\", placeholder='Text Here...'), \n",
+    "                                        gr.inputs.Number(label=\"Positions\")], \n",
+    "                    outputs=\"text\", \n",
+    "                    title=\"ReRAM Paragraph Classification\", allow_flagging=False,\n",
+    "                    examples=[[example_paragraph, example_position]],\n",
+    "                    )\n",
+    "\n",
+    "demo.launch(share=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.15 ('nlp')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.15"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "a0944428a9b48e048108e25849b0259875c53ceed2bb9cda9ef2b8036da8c8e0"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

extraction.py

@@ -0,0 +1,22 @@
+try:
+    from .src.run import NLP_classification
+except:
+    from src.run import NLP_classification
+import yaml
+import os
+
+
+class paragraph_extract:
+    def __init__(self):
+        config_file = 'configs/config.yaml'
+        config_file = os.path.join(os.path.dirname(__file__), config_file)
+        self.config = yaml.load(open(config_file), Loader=yaml.FullLoader)
+        self.config['extract']['selected_model'] = os.path.join(os.path.dirname(__file__), self.config['extract']['selected_model'])
+        
+        self.runner = NLP_classification(**self.config['model'])
+
+    def extract(self, paragraphs, positions):
+
+        labels = self.runner.label_extraction(paragraphs, positions, **self.config['extract'])
+        
+        return labels

src/clustering.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import numpy as np
+import seaborn as sns
+from matplotlib import pyplot as plt
+import umap
+
+def dim_reduction(target_embeddings, umap_dim=2, n_neighbors=15, min_dist=0.1):
+    """
+    Dimension reduction using UMAP.
+    """
+    reducer = umap.UMAP(n_neighbors=n_neighbors, n_components=umap_dim, min_dist=min_dist, metric='cosine', random_state=500)
+    embeddings = reducer.fit_transform(target_embeddings)
+    return embeddings
+
+
+def clustering_plot(target_label, embeddings, label_trues, model_preds=None, umap_dim=2, n_neighbors=15, min_dist=0.1):
+    """
+    Plot the clustering results.
+    """
+    label_dict = {0:'Abstract', 1:'Introduction', 2:'Main', 3:'Methods', 4:'Summary', 5:'Captions'}
+    
+    target_index = np.where(label_trues == target_label)[0]
+    
+    trues = label_trues[target_index]
+    embeddings = embeddings[target_index]
+    
+    embeddings = dim_reduction(embeddings, umap_dim=umap_dim, n_neighbors=n_neighbors, min_dist=min_dist)
+    
+    df = pd.DataFrame(embeddings, columns=['x', 'y'])
+    df['true'] = trues
+    df['true'] = df['true'].map(label_dict)
+    if model_preds is not None:
+        df['pred'] = model_preds[target_index]
+        df['pred'] = df['pred'].map(label_dict)
+        
+    sns.scatterplot(x='x', y='y', hue='true', data=df, palette='Set2')
+    plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
+    plt.show()
+    
+    if model_preds is not None:
+        sns.scatterplot(x='x', y='y', hue='pred', data=df, palette='Set2')
+        plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
+        plt.show()
+        
+    return df
+    
+    
+    

src/datas.py

@@ -0,0 +1,133 @@
+import numpy as np
+
+import pandas as pd
+import pickle
+from sklearn.model_selection import train_test_split
+import torch
+import os
+import ast
+from sklearn.utils import shuffle
+import random
+from spacy.lang.en import English
+from .utils import sentencepiece
+
+def make_dataset(csv_file, tokenizer, max_length=512, padding=None, random_state=1000, data_cut=None, sentence_piece=True):
+        ''' data load '''
+        ''' 1기+2기 데이터 '''
+        #data = csv_file
+        #total_data = pd.read_csv(data)
+
+        ''' 재선이형이 준 데이터 '''
+        total_data = pd.read_csv(csv_file)
+        total_data.columns=['paragraph', 'category', 'position', 'portion']
+        label_dict = {'Abstract':0, 'Introduction':1, 'Main':2, 'Methods':3, 'Summary':4, 'Captions':5}
+        total_data['label'] = total_data.category.replace(label_dict)
+        
+        if not data_cut is None:
+            total_data = total_data.iloc[:data_cut,:]
+            
+        total_text = total_data['paragraph'].to_list()
+        total_label = total_data['label'].to_list()
+        total_position = total_data['position'].to_list()
+        total_portion = total_data['portion'].to_list()
+
+        ''' type error 방지 '''
+        if type(total_label[0]) == str:
+            total_label = [ast.literal_eval(l) for l in total_label]
+
+        if type(total_label[0]) == int:
+            total_label = np.eye(6)[total_label].tolist()
+
+        train_text, val_text, train_labels, val_labels, train_position, val_position, train_portion, val_portion = train_test_split(total_text, total_label, total_position, total_portion, test_size=0.2, random_state=random_state, stratify=total_label)
+
+        ''' data들 tokenizing '''
+        if not sentence_piece:
+            train_encodings= tokenizer.batch_encode_plus(train_text, truncation=True, return_token_type_ids=True, max_length=max_length, add_special_tokens=True, return_attention_mask=True, padding='max_length')
+            val_encodings = tokenizer.batch_encode_plus(val_text, truncation=True, return_token_type_ids=True, max_length=max_length, add_special_tokens=True, return_attention_mask=True, padding='max_length')
+        else:
+            nlp = English()
+            nlp.add_pipe('sentencizer')
+            train_encodings = sentencepiece(train_text, nlp, tokenizer, max_length=max_length)
+            val_encodings = sentencepiece(val_text, nlp, tokenizer, max_length=max_length)
+
+        ''' token tensor 화 '''
+        train_encodings = {key: torch.tensor(val) for key, val in train_encodings.items()}
+        val_encodings = {key: torch.tensor(val) for key, val in val_encodings.items()}
+
+        ''' labels tensor 화 '''
+        train_labels_ = {}
+        train_labels_['label_onehot'] = torch.tensor(train_labels, dtype=torch.float)
+        train_labels_['label'] = torch.tensor([t.index(1) for t in train_labels], dtype=torch.int)
+        train_labels = train_labels_
+
+        val_labels_ = {}
+        val_labels_['label_onehot'] = torch.tensor(val_labels, dtype=torch.float)
+        val_labels_['label'] = torch.tensor([t.index(1) for t in val_labels], dtype=torch.long)
+        val_labels = val_labels_
+
+        ''' position tensor 화 '''
+        train_positions_ = {}
+        train_positions_['position'] = torch.tensor(train_position, dtype=torch.float)
+        train_positions_['portion'] = torch.tensor(train_portion, dtype=torch.float)
+        train_positions = train_positions_
+        
+        val_positions_ = {}
+        val_positions_['position'] = torch.tensor(val_position, dtype=torch.float)
+        val_positions_['portion'] = torch.tensor(val_portion, dtype=torch.float)
+        val_positions = val_positions_
+
+        ''' dataset class 생성 '''
+        class CustomDataset(torch.utils.data.Dataset):
+            def __init__(self, encodings, labels, texts, positions):
+                self.encodings = encodings
+                self.labels = labels
+                self.texts = texts
+                self.positions = positions
+
+            def __getitem__(self, idx):
+                item = {key: val[idx] for key, val in self.encodings.items()}
+                item['text'] = self.texts[idx]
+                # scalar version
+                item['label'] = self.labels['label'][idx]
+                # one-hot version
+                item['label_onehot'] = self.labels['label_onehot'][idx]
+                # position
+                item['position'] = self.positions['position'][idx]
+                #portion
+                item['portion'] = self.positions['portion'][idx]
+                return item
+
+            def __len__(self):
+                return len(self.labels['label_onehot'])
+
+        ''' train을 위한 format으로 data들 변환 '''
+        train_dataset = CustomDataset(train_encodings, train_labels, train_text, train_positions)
+        val_dataset = CustomDataset(val_encodings, val_labels, val_text, val_positions)
+        
+        return train_dataset, val_dataset
+
+
+def make_extract_dataset(paragraphs, positions, tokenizer, max_length):
+    encodings = tokenizer.batch_encode_plus(paragraphs, truncation=True, return_token_type_ids=True, max_length=max_length, add_special_tokens=True, return_attention_mask=True, padding='max_length', return_tensors='pt')
+    positions_ = {}
+    positions_['position'] = torch.tensor(positions, dtype=torch.float)
+    positions = positions_
+    
+    class CustomDataset(torch.utils.data.Dataset):
+        def __init__(self, encodings, positions):
+            self.encodings = encodings
+            self.positions = positions
+
+        def __getitem__(self, idx):
+            item = {key: val[idx] for key, val in self.encodings.items()}
+            # position
+            item['position'] = self.positions['position'][idx]
+            return item
+
+        def __len__(self):
+            return len(self.encodings['input_ids'])
+    
+    dataset = CustomDataset(encodings, positions)
+    return dataset
+
+    

src/model.py

@@ -0,0 +1,225 @@
+import torch
+from torch.nn import functional as F
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from torch import nn
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
+from torch.nn import Identity
+from transformers.activations import get_activation
+import numpy as np
+#from torch_scatter import scatter_add
+from .utils import input_check, pos_encoding
+
+class classification_model(torch.nn.Module):
+    def __init__(self, pretrained_model, config, num_classifier=1, num_pos_emb_layer=1, bertsum=False, device=None):
+        super(classification_model, self).__init__()
+        
+        self.config = config
+        self.num_labels = config.num_labels
+        self.pretrained_model = pretrained_model
+        if hasattr(config, 'd_model'):
+            self.pretrained_hidden = config.d_model
+        elif hasattr(config, 'hidden_size'):
+            self.pretrained_hidden = config.hidden_size
+        self.sequence_summary = SequenceSummary(config)
+        self.bertsum = bertsum
+        self.device = device
+        self.return_hidden = False
+        self.return_hidden_pretrained = False
+        
+        if self.bertsum:
+            #self.pooling_1 = GATpooling(self.pretrained_hidden)
+            #self.fnn_1 = nn.Linear(self.pretrained_hidden, self.pretrained_hidden)
+            self.pooling_2 = GATpooling(self.pretrained_hidden, self.device)
+            self.fnn_2 = nn.Linear(self.pretrained_hidden, self.pretrained_hidden)
+
+        self.pos_emb_layer = nn.Sequential(*[nn.Linear(self.pretrained_hidden, self.pretrained_hidden) for _ in range(num_pos_emb_layer)])
+
+        dim_list = np.linspace(self.pretrained_hidden, config.num_labels, num_classifier+1, dtype=np.int32)
+        #dim_list = np.linspace(768, config.num_labels, num_classifier+1, dtype=np.int32)
+        self.classifiers = nn.ModuleList()
+        for c in range(num_classifier):
+            self.classifiers.append(nn.Linear(dim_list[c], dim_list[c+1]))
+        
+    def forward(self, inputs):
+        hidden_states = None
+        input_ids = inputs['input_ids']
+        token_type_ids = inputs['token_type_ids']
+        attention_mask = inputs['attention_mask']
+        position = inputs['position']
+        transformer_inputs = input_check({'input_ids':input_ids, 'token_type_ids':token_type_ids, 'attention_mask':attention_mask}, self.pretrained_model)
+        
+        pretrianed_output = self.pretrained_model(**transformer_inputs)
+        output = pretrianed_output[0]
+        
+        if self.return_hidden_pretrained and self.return_hidden:
+            hidden_states = pretrianed_output[1]
+        if self.bertsum:
+            output = scatter_add(output, inputs['sentence_batch'], dim=-2)
+            #output = self.pooling_1(output, inputs['sentence_batch'])
+            #output = self.fnn_1(output)
+            output = self.pooling_2(output)
+            output = output.squeeze()
+            output = self.fnn_2(output)
+        else:
+            output = self.sequence_summary(output)
+            
+        # paragraph positional encoding vector add
+        pos_emb = pos_encoding(position, self.pretrained_hidden).to(self.device, dtype=torch.float)
+        output = torch.add(output,pos_emb)
+        output = self.pos_emb_layer(output)
+            
+        if self.return_hidden and not self.return_hidden_pretrained:
+            hidden_states = output
+        for layer in self.classifiers:
+            output = layer(output)
+        
+        logits = output
+        
+        if 'labels' in inputs.keys():
+            loss = self.classification_loss_f(inputs, logits)
+        else:
+            loss = None
+        
+        return loss, output, hidden_states
+    
+    def classification_loss_f(self, inputs, logits):
+        labels=inputs['labels']
+        loss=None
+        
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+        return loss
+
+
+class GATpooling(nn.Module):
+    def __init__(self, hidden_size, device=None):
+        super(GATpooling, self).__init__()
+        self.gate_nn = nn.Linear(hidden_size, 1)
+        self.device = device
+        
+    def forward(self, x, batch=None):
+        if batch==None:
+            batch = torch.zeros(x.shape[-2], dtype=torch.long).to(self.device)
+        gate = self.gate_nn(x)
+        gate = F.softmax(gate, dim=-1)
+        out = scatter_add(gate*x, batch, dim=-2)
+        return out
+    
+
+class SequenceSummary(nn.Module):
+    r"""
+    Compute a single vector summary of a sequence hidden states.
+    Args:
+        config ([`PretrainedConfig`]):
+            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
+            config class of your model for the default values it uses):
+            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:
+                - `"last"` -- Take the last token hidden state (like XLNet)
+                - `"first"` -- Take the first token hidden state (like Bert)
+                - `"mean"` -- Take the mean of all tokens hidden states
+                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
+                - `"attn"` -- Not implemented now, use multi-head attention
+            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
+            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
+              (otherwise to `config.hidden_size`).
+            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
+              another string or `None` will add no activation.
+            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
+            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.summary_type = getattr(config, "summary_type", "mean")
+        if self.summary_type == "attn":
+            # We should use a standard multi-head attention module with absolute positional embedding for that.
+            # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276
+            # We can probably just use the multi-head attention module of PyTorch >=1.1.0
+            raise NotImplementedError
+
+        self.summary = Identity()
+        if hasattr(config, "summary_use_proj") and config.summary_use_proj:
+            if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0:
+                num_classes = config.num_labels
+            else:
+                num_classes = config.hidden_size
+            self.summary = nn.Linear(config.hidden_size, num_classes)
+
+        activation_string = getattr(config, "summary_activation", None)
+        self.activation: Callable = get_activation(activation_string) if activation_string else Identity()
+
+        self.first_dropout = Identity()
+        if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0:
+            self.first_dropout = nn.Dropout(config.summary_first_dropout)
+
+        self.last_dropout = Identity()
+        if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0:
+            self.last_dropout = nn.Dropout(config.summary_last_dropout)
+
+    def forward(
+        self, hidden_states: torch.FloatTensor, cls_index: Optional[torch.LongTensor] = None
+    ) -> torch.FloatTensor:
+        """
+        Compute a single vector summary of a sequence hidden states.
+        Args:
+            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
+                The hidden states of the last layer.
+            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
+                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
+        Returns:
+            `torch.FloatTensor`: The summary of the sequence hidden states.
+        """
+        if self.summary_type == "last":
+            output = hidden_states[:, -1]
+        elif self.summary_type == "first":
+            output = hidden_states[:, 0]
+        elif self.summary_type == "mean":
+            output = hidden_states.mean(dim=1)
+        elif self.summary_type == "cls_index":
+            if cls_index is None:
+                cls_index = torch.full_like(
+                    hidden_states[..., :1, :],
+                    hidden_states.shape[-2] - 1,
+                    dtype=torch.long,
+                )
+            else:
+                cls_index = cls_index.unsqueeze(-1).unsqueeze(-1)
+                cls_index = cls_index.expand((-1,) * (cls_index.dim() - 1) + (hidden_states.size(-1),))
+            # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states
+            output = hidden_states.gather(-2, cls_index).squeeze(-2)  # shape (bsz, XX, hidden_size)
+        elif self.summary_type == "attn":
+            raise NotImplementedError
+
+        output = self.first_dropout(output)
+        output = self.summary(output)
+        output = self.activation(output)
+        output = self.last_dropout(output)
+
+        return output
+
+
+
+    
+
+
+

src/run.py

@@ -0,0 +1,279 @@
+from glob import glob
+from tqdm import tqdm
+import numpy as np
+import pickle
+from sklearn.model_selection import train_test_split
+import torch
+import os
+import ast
+from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
+from transformers import EarlyStoppingCallback
+from transformers import AutoConfig, AutoModel, AutoTokenizer, AutoModelForSequenceClassification
+from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
+from sklearn.utils import shuffle
+from transformers import get_cosine_schedule_with_warmup
+from torch.nn import functional as F
+import random
+import pandas as pd
+from .datas import make_dataset, make_extract_dataset
+from .utils import set_seed, accuracy_per_class, compute_metrics, model_eval, checkpoint_save, EarlyStopping, model_freeze, get_hidden
+from .model import classification_model
+from transformers import BigBirdTokenizer
+import transformers
+
+class NLP_classification():
+    def __init__(self, model_name=None, data_file=None, max_length=None, random_state=1000, task_type='onehot', freeze_layers=None, num_classifier=1, num_pos_emb_layer=1, gpu_num=0, sentence_piece=True, bertsum=False):
+        self.model_name = model_name
+        self.data_file = data_file
+        self.max_length = max_length
+        self.random_state = random_state
+        self.task_type = task_type
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name, do_lower_case=False)
+        if model_name == 'google/bigbird-roberta-base':
+            self.tokenizer = BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')
+        self.config = AutoConfig.from_pretrained(model_name, num_labels=6)
+            
+        #self.pretrained_model = AutoModelForSequenceClassification.from_config(self.config)
+        self.pretrained_model = AutoModel.from_config(self.config)
+        self.freeze_layers=freeze_layers
+        self.num_classifier=num_classifier
+        self.num_pos_emb_layer=num_pos_emb_layer
+        self.gpu_num=gpu_num
+        self.sentence_piece=sentence_piece
+        self.bertsum=bertsum
+        if self.max_length is None:
+            self.padding='longest'
+        else:
+            self.padding='max_length'
+        
+        
+    def training(self, epochs=50, batch_size=4, lr=1e-5, dropout=0.1, data_cut=None, early_stop_count=10, 
+                wandb_log=False, wandb_project=None, wandb_group=None, wandb_name=None, wandb_memo=None):
+        os.environ["CUDA_VISIBLE_DEVICES"]= "{0}".format(int(self.gpu_num))
+        #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        device = torch.device('cuda:{0}'.format(int(self.gpu_num)))
+        torch.cuda.set_device(device)
+        set_seed(self.random_state)
+        torch.set_num_threads(10)
+        
+        if wandb_log is True:
+            import wandb
+            wandb.init(project=wandb_project, reinit=True, group=wandb_group, notes=wandb_memo)
+            wandb.run.name = wandb_name
+            wandb.run.save()
+            parameters = wandb.config
+            parameters.lr = lr
+            parameters.batch_size = batch_size
+            parameters.dropout = dropout
+            parameters.train_num = data_cut
+            parameters.max_length = self.max_length
+            parameters.model_name = self.model_name
+            parameters.task_type = self.task_type
+        
+        '''data loading'''
+        train_dataset, val_dataset = make_dataset(csv_file=self.data_file, tokenizer=self.tokenizer, max_length=self.max_length, padding=self.padding, random_state=self.random_state, data_cut=data_cut, sentence_piece=self.sentence_piece)
+        
+        '''loader making'''
+        train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=RandomSampler(train_dataset))
+        val_loader = DataLoader(val_dataset, batch_size=batch_size, sampler=SequentialSampler(val_dataset))
+
+        ''' model load '''
+        model=classification_model(self.pretrained_model, self.config, num_classifier=self.num_classifier, num_pos_emb_layer=self.num_pos_emb_layer, bertsum=self.bertsum, device=device)
+        model=model_freeze(model, self.freeze_layers)
+        model.to(device)
+        
+        ''' running setting '''
+        loss_fn = torch.nn.BCEWithLogitsLoss()
+        optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr, eps=1e-8)
+        scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=100, num_training_steps=(len(train_loader)*epochs))
+        early_stopping = EarlyStopping(patience = early_stop_count, verbose = True)
+        
+        ''' running '''
+        best_epoch = None
+        best_val_f1 = None
+
+        for epoch in range(epochs):
+            model.train()
+            loss_all = 0
+            step = 0
+
+            for data in tqdm(train_loader):
+                input_ids=data['input_ids'].to(device, dtype=torch.long)
+                mask = data['attention_mask'].to(device, dtype=torch.long)
+                token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)
+                if self.task_type=='onehot':
+                    targets=data['label_onehot'].to(device, dtype=torch.float)
+                elif self.task_type=='scalar':
+                    targets=data['label'].to(device, dtype=torch.long)
+                position = data['position']
+                inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+                  'labels': targets, 'position': position}
+                if self.sentence_piece:
+                    sentence_batch = data['sentence_batch'].to(device, dtype=torch.long)
+                    inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+              'labels': targets, 'sentence_batch': sentence_batch, 'position': position}
+
+                outputs = model(inputs)
+                output = outputs[1]
+                loss = outputs[0]
+
+                optimizer.zero_grad()
+                #loss=loss_fn(output, targets)
+                loss_all += loss.item()
+
+                loss.backward()
+                optimizer.step()
+                scheduler.step()
+                #print(optimizer.param_groups[0]['lr'])
+
+            train_loss = loss_all/len(train_loader)
+            val_loss, val_acc, val_precision, val_recall, val_f1 = model_eval(model, device, val_loader, task_type=self.task_type, sentence_piece=self.sentence_piece)
+            
+            if wandb_log is True:
+                wandb.log({'train_loss':train_loss, 'val_loss':val_loss, 'val_acc':val_acc,
+                           'val_precision':val_precision, 'val_recall':val_recall, 'val_f1':val_f1})
+
+            if best_val_f1 is None or val_f1 >= best_val_f1:
+                best_epoch = epoch+1
+                best_val_f1 = val_f1
+                checkpoint_save(model, val_f1, wandb_name=wandb_name)
+                
+            print('Epoch: {:03d}, Train Loss: {:.7f}, Val Loss: {:.7f}, Val Acc: {:.7f}, Val Precision: {:.7f}, Val Recall: {:.7f}, Val F1: {:.7f} '.format(epoch+1, train_loss, val_loss, val_acc, val_precision, val_recall, val_f1))
+            
+            early_stopping(val_f1)
+            if early_stopping.early_stop:
+                print("Early stopping")
+                break
+                
+        wandb.finish()
+        
+        
+    def prediction(self, selected_model=None, batch_size=8):
+        os.environ["CUDA_VISIBLE_DEVICES"]= "{0}".format(int(self.gpu_num))
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        set_seed(self.random_state)
+        torch.set_num_threads(10)
+        task_type=self.task_type
+        
+        '''data loading'''
+        train_dataset, val_dataset = make_dataset(csv_file=self.data_file, tokenizer=self.tokenizer, max_length=self.max_length, padding=self.padding, random_state=self.random_state, data_cut=None, sentence_piece=self.sentence_piece)
+    
+        '''loader making'''
+        train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=RandomSampler(train_dataset))
+        val_loader = DataLoader(val_dataset, batch_size=batch_size, sampler=SequentialSampler(val_dataset))
+        
+        ''' model load '''
+        model=classification_model(self.pretrained_model, self.config, num_classifier=self.num_classifier, num_pos_emb_layer=self.num_pos_emb_layer, bertsum=self.bertsum, device=device)
+        model.load_state_dict(torch.load(selected_model))
+        model.to(device)
+        
+        ''' prediction '''
+        print('start trainset prediction')
+        train_results = model_eval(model, device, train_loader, task_type=self.task_type, return_values=True, sentence_piece=self.sentence_piece)
+        print('start evalset prediction')
+        eval_results = model_eval(model, device, val_loader, task_type=self.task_type, return_values=True, sentence_piece=self.sentence_piece)
+        
+        print('train result: acc:{0} | precision:{1} | recall:{2} | f1:{3}'.format(train_results[1], train_results[2], train_results[3], train_results[4]))
+        print('eval result: acc:{0} | precision:{1} | recall:{2} | f1:{3}'.format(eval_results[1], eval_results[2], eval_results[3], eval_results[4]))
+        
+        total_text = train_results[7] + eval_results[7]
+        total_out = train_results[6] + eval_results[6]
+        total_target = train_results[5] + eval_results[5]
+        
+        if self.task_type == 'onehot':
+            total_out = [i.argmax() for i in total_out]
+            total_target = [i.argmax() for i in total_target]
+
+        total_data = {'text':total_text, 'label':total_target, 'predict':total_out}
+        total_df = pd.DataFrame(total_data)
+        
+        ''' result return '''
+        return total_df
+    
+    
+    def get_embedding(self, selected_model=None, batch_size=8, return_hidden=True, return_hidden_pretrained=False):
+        os.environ["CUDA_VISIBLE_DEVICES"]= "{0}".format(int(self.gpu_num))
+        #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        device = torch.device('cuda:{0}'.format(int(self.gpu_num)))
+        torch.cuda.set_device(device)
+        set_seed(self.random_state)
+        torch.set_num_threads(10)
+        task_type=self.task_type
+        
+        '''data loading'''
+        train_dataset, val_dataset = make_dataset(csv_file=self.data_file, tokenizer=self.tokenizer, max_length=self.max_length, padding=self.padding, random_state=self.random_state, data_cut=None, sentence_piece=self.sentence_piece)
+    
+        '''loader making'''
+        train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=RandomSampler(train_dataset))
+        val_loader = DataLoader(val_dataset, batch_size=batch_size, sampler=SequentialSampler(val_dataset))
+        
+        ''' model load '''
+        model=classification_model(self.pretrained_model, self.config, num_classifier=self.num_classifier, num_pos_emb_layer=self.num_pos_emb_layer, bertsum=self.bertsum, device=device)
+        model.return_hidden = return_hidden
+        model.return_hidden_pretrained = return_hidden_pretrained
+        if selected_model is not None:
+            model.load_state_dict(torch.load(selected_model))
+        model.to(device)
+        
+        ''' get hidden '''
+        print('start make hidden states (trainset)')
+        train_hiddens, train_targets = get_hidden(model, device, train_loader, task_type=self.task_type, sentence_piece=self.sentence_piece)
+        print('start evalset prediction (eval set)')
+        eval_hiddens, eval_targets = get_hidden(model, device, val_loader, task_type=self.task_type, sentence_piece=self.sentence_piece)
+        total_hiddens = np.array(train_hiddens + eval_hiddens)
+        total_targets = np.array(train_targets + eval_targets)
+        
+        
+        return total_hiddens, total_targets
+        
+
+    def label_extraction(self, paragraphs, positions, selected_model=None, batch_size=16):
+        label_dict = {'Abstract':0, 'Introduction':1, 'Main':2, 'Methods':3, 'Summary':4, 'Captions':5}
+        #os.environ["CUDA_VISIBLE_DEVICES"]= "{0}".format(int(self.gpu_num))
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        set_seed(self.random_state)
+        torch.set_num_threads(10)
+        
+        ''' data to list '''
+        is_list = True
+        if not isinstance(paragraphs, list):
+            paragraphs = [paragraphs]
+            is_list = False
+        if not isinstance(positions, list):
+            positions = [positions]
+            is_list = False
+        
+        '''data encoding'''
+        dataset = make_extract_dataset(paragraphs, positions, tokenizer=self.tokenizer, max_length=self.max_length)
+
+        '''loader making'''
+        data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
+        
+        ''' model load '''
+        model=classification_model(self.pretrained_model, self.config, num_classifier=self.num_classifier, num_pos_emb_layer=self.num_pos_emb_layer, bertsum=self.bertsum, device=device)
+        model.load_state_dict(torch.load(selected_model))
+        model.to(device)
+        
+        ''' prediction '''
+        model.eval()
+        predicts = []
+        with torch.no_grad():
+            for batch in tqdm(data_loader):
+                inputs  = {}
+                inputs['input_ids'] = batch['input_ids'].to(device)
+                inputs['attention_mask'] = batch['attention_mask'].to(device)
+                inputs['token_type_ids'] = batch['token_type_ids'].to(device)
+                inputs['position'] = batch['position']
+                outputs = model(inputs)
+                logits = outputs[1]
+                logits = logits.detach().cpu().numpy()
+                logits = logits.argmax(axis=1).flatten()
+                logits = logits.tolist()
+                predicts.extend(logits)
+        predicts = [list(label_dict.keys())[list(label_dict.values()).index(i)] for i in predicts]
+        
+        if not is_list:
+            predicts = predicts[0]
+        return predicts
+
+        

src/utils.py

@@ -0,0 +1,299 @@
+from doctest import DocFileCase
+from tqdm import tqdm
+import numpy as np
+import torch
+from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
+from sklearn.utils import shuffle
+import random
+import datetime as dt
+import os
+from glob import glob
+from spacy.lang.en import English
+import inspect
+
+def checkpoint_save(model, val_loss, checkpoint_dir=None, wandb_name=None):
+    if checkpoint_dir is None:
+        checkpoint_dir = './save_model'
+    if not os.path.isdir(checkpoint_dir):
+        os.mkdir(checkpoint_dir)
+    x = dt.datetime.now()
+    y = x.year
+    m = x.month
+    d = x.day
+    
+    if wandb_name is None:
+        wandb_name = "testing"
+    
+    torch.save(model.state_dict(), "./save_model/{}_{}_{}_{:.4f}_{}.pt".format(y, m, d, val_loss, wandb_name))
+    
+    #saved_dict_list = glob(os.path.join(checkpoint_dir, '*.pt'))
+    saved_dict_list = glob(os.path.join(checkpoint_dir, '{}_{}_{}_*_{}.pt'.format(y,m,d,wandb_name)))
+    
+    
+    val_loss_list = np.array([float(os.path.basename(loss).split("_")[3]) for loss in saved_dict_list])
+    saved_dict_list.pop(val_loss_list.argmax())
+    
+    for i in saved_dict_list:
+        os.remove(i)
+
+
+def set_seed(seed):
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+
+def accuracy_per_class(preds, labels):
+    label_dict = {'Abstract':0, 'Intro':1, 'Main':2, 'Method':3, 'Summary':4, 'Caption':5}
+    label_dict_inverse = {v: k for k, v in label_dict.items()}
+    
+    class_list = []
+    acc_list = []
+    for label in list(label_dict.values()):
+        y_preds = preds[labels==label]
+        y_true = labels[labels==label]
+        class_list.append(label_dict_inverse[label])
+        acc_list.append("{0}/{1}".format(len(y_preds[y_preds==label]), len(y_true)))
+    
+    print("{:10} {:10} {:10} {:10} {:10} {:10}".format(class_list[0], class_list[1], class_list[2], class_list[3], class_list[4], class_list[5]))
+    print("{:10} {:10} {:10} {:10} {:10} {:10}".format(acc_list[0], acc_list[1], acc_list[2], acc_list[3], acc_list[4], acc_list[5]))    
+
+
+def compute_metrics(output, target, task_type='onehot'):
+    if task_type=='onehot':
+        pred=np.argmax(output, axis=1).flatten()
+        labels=np.argmax(target, axis=1).flatten()
+    elif task_type=='scalar':
+        pred=np.argmax(output, axis=1).flatten()
+        labels=np.array(target).flatten()
+    accuracy = accuracy_score(y_true=labels, y_pred=pred)
+    recall = recall_score(y_true=labels, y_pred=pred, average='macro')
+    precision = precision_score(y_true=labels, y_pred=pred, average='macro', zero_division=0)
+    f1 = f1_score(y_true=labels, y_pred=pred, average='macro')
+    
+    accuracy_per_class(pred, labels)
+        
+    return [accuracy, precision, recall, f1]
+
+def input_check(input_dict, model):
+    model_inputs = inspect.signature(model.forward).parameters.keys()
+    inputs = {}
+    for key, val in input_dict.items():
+        if key in model_inputs:
+            inputs[key] = val
+    return inputs
+    
+    
+
+def model_eval(model, device, loader, task_type='onehot', return_values=False, sentence_piece=False):
+    model.eval()
+    error = 0
+    accuracy = 0
+    precision = 0
+    recall = 0
+    f1 = 0
+    eval_targets=[]
+    eval_outputs=[]
+    eval_texts=[]
+    with torch.no_grad():
+        for data in tqdm(loader):
+            eval_texts.extend(data['text'])
+            input_ids=data['input_ids'].to(device, dtype=torch.long)
+            mask = data['attention_mask'].to(device, dtype=torch.long)
+            token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)
+            if task_type=='onehot':
+                targets=data['label_onehot'].to(device, dtype=torch.float)
+            elif task_type=='scalar':
+                targets=data['label'].to(device, dtype=torch.long)
+            position = data['position']
+            inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+          'labels': targets, 'position': position}
+            if sentence_piece:
+                sentence_batch = data['sentence_batch'].to(device, dtype=torch.long)
+                inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+          'labels': targets, 'sentence_batch': sentence_batch, 'position': position}
+            outputs = model(inputs) 
+            output = outputs[1]
+            loss = outputs[0]
+            #loss=loss_fn(output, targets)
+            error+=loss
+            #output = torch.sigmoid(output)
+            eval_targets.extend(targets.detach().cpu().numpy())
+            eval_outputs.extend(output.detach().cpu().numpy())
+            
+    error = error / len(loader)
+    accuracy, precision, recall, f1 = compute_metrics(eval_outputs, eval_targets, task_type=task_type)
+    
+    if return_values:
+        return [error, accuracy, precision, recall, f1, eval_targets, eval_outputs, eval_texts]
+    else:
+        return [error, accuracy, precision, recall, f1]
+    
+    
+def get_hidden(model, device, loader, task_type='onehot', sentence_piece=False):
+    model.eval()
+    total_hidden_state = []
+    total_targets=[]
+    with torch.no_grad():
+        for data in tqdm(loader):
+            input_ids=data['input_ids'].to(device, dtype=torch.long)
+            mask = data['attention_mask'].to(device, dtype=torch.long)
+            token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)
+            if task_type=='onehot':
+                targets=data['label_onehot'].to(device, dtype=torch.float)
+            elif task_type=='scalar':
+                targets=data['label'].to(device, dtype=torch.long)
+            position = data['position']
+            inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+          'labels': targets, 'position': position}
+            if sentence_piece:
+                sentence_batch = data['sentence_batch'].to(device, dtype=torch.long)
+                inputs = {'input_ids': input_ids, 'attention_mask': mask, 'token_type_ids': token_type_ids, 
+          'labels': targets, 'sentence_batch': sentence_batch, 'position': position}
+            outputs = model(inputs) 
+            hidden_state = outputs[2]
+            total_hidden_state.extend(hidden_state.detach().cpu().numpy())
+            total_targets.extend(targets.detach().cpu().numpy())
+    return total_hidden_state, total_targets
+            
+
+
+def sentencepiece(paragraph_list, spacy_nlp, tokenizer, max_length=512):
+    # 현재 token type ids가 tokenizer에서 생성하는 데이터가 아닌 내가 임의적으로 0, 1로만 넣도록 해놓았음, XLNET 같은건 CLS가 2로 되는 경우 같이 이 규칙을 벗어나는 경우가 있어서 나중에 문제되면 수정 필요
+    encode_datas = {'input_ids': [], 'token_type_ids': [], 'attention_mask': [], 'sentence_batch': []}
+    for paragraph in paragraph_list:
+        doc = spacy_nlp(paragraph)
+        sentence_encode = [sent.text for sent in doc.sents]
+        sentence_encode = tokenizer.batch_encode_plus(sentence_encode, max_length=max_length, padding='max_length', return_attention_mask=True, return_token_type_ids=True)
+
+        sentence_list = sentence_encode['input_ids']
+        mask_list = sentence_encode['attention_mask']
+        pad_token = None
+        pad_position = None
+        total_sentence = torch.tensor([], dtype=torch.int)
+        token_type_ids = []
+        s_batch = []
+        
+        for n, s in enumerate(sentence_list):
+            if pad_token is None:
+                pad_token = s[mask_list[n].index(0)]
+            if pad_position is None:
+                if s[0] == pad_token:
+                    pad_position = 'start'
+                else:
+                    pad_position = 'end'
+
+            s=torch.tensor(s, dtype=torch.int)
+            s = s[s!=pad_token]
+            total_length = len(total_sentence) + len(s)
+            if total_length > max_length:
+                break
+            total_sentence = torch.concat([total_sentence, s])
+            token_type_ids = token_type_ids + [n%2]*len(s)
+            s_batch = s_batch + [n]*len(s)
+            
+        total_sentence = total_sentence.tolist()
+        pad_length = max_length - len(total_sentence)
+        attention_mask = [1]*len(total_sentence)
+        if pad_position == 'end':
+            total_sentence = total_sentence + [pad_token]*pad_length
+            attention_mask = attention_mask + [0]*pad_length
+            s_batch = s_batch + [max(s_batch)+1]*pad_length
+            if n%2 == 0:
+                token_type_ids = token_type_ids + [1]*pad_length
+            else:
+                token_type_ids = token_type_ids + [0]*pad_length
+
+        elif pad_position == 'start':
+            total_sentence = [pad_token]*pad_length + total_sentence
+            attention_mask = [0]*pad_length + attention_mask
+            s_batch = [max(s_batch)+1]*pad_length + s_batch
+            if n%2 == 0:
+                token_type_ids = [0]*pad_length + token_type_ids
+            else:
+                token_type_ids = [1]*pad_length + token_type_ids
+
+        encode_datas['input_ids'].append(total_sentence)
+        encode_datas['token_type_ids'].append(token_type_ids)
+        encode_datas['attention_mask'].append(attention_mask)
+        encode_datas['sentence_batch'].append(s_batch)
+    
+    return encode_datas
+    
+    
+class EarlyStopping:
+    """주어진 patience 이후로 validation loss가 개선되지 않으면 학습을 조기 중지"""
+    def __init__(self, patience=7, verbose=False, delta=0):
+        """
+        Args:
+            patience (int): validation loss가 개선된 후 기다리는 기간
+                            Default: 7
+            verbose (bool): True일 경우 각 validation loss의 개선 사항 메세지 출력
+                            Default: False
+            delta (float): 개선되었다고 인정되는 monitered quantity의 최소 변화
+                            Default: 0
+        """
+        self.patience = patience
+        self.verbose = verbose
+        self.counter = 0
+        self.best_score = None
+        self.early_stop = False
+        self.f1_score_max = 0.
+        self.delta = delta
+
+    def __call__(self, f1_score):
+
+        score = -f1_score
+
+        if self.best_score is None:
+            self.best_score = score
+            self.save_checkpoint(f1_score)
+        elif score > self.best_score + self.delta:
+            self.counter += 1
+            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_score = score
+            self.save_checkpoint(f1_score)
+            self.counter = 0
+
+    def save_checkpoint(self, f1_score):
+        '''validation loss가 감소하면 감소를 출력한다.'''
+        if self.verbose:
+            print(f'F1 score increase ({self.f1_score_max:.6f} --> {f1_score:.6f}). ')
+        self.f1_score_max = f1_score
+
+
+def model_freeze(model, freeze_layers=None):
+    if freeze_layers == 0:
+        return model
+    
+    if freeze_layers is not None:
+        for param in model.pretrained_model.base_model.word_embedding.parameters():
+            param.requires_grad = False
+
+        if freeze_layers != -1:
+            # if freeze_layer_count == -1, we only freeze the embedding layer
+            # otherwise we freeze the first `freeze_layer_count` encoder layers
+            for layer in model.pretrained_model.base_model.layer[:freeze_layers]:
+                for param in layer.parameters():
+                    param.requires_grad = False                 
+    return model
+
+def pos_encoding(pos, d, n=10000):
+    encoding_list = []
+    for p in pos:
+        P = np.zeros(d)
+        for i in np.arange(int(d/2)):
+            denominator = np.power(n, 2*i/d)
+            P[2*i] = np.sin(p/denominator)
+            P[2*i+1] = np.cos(p/denominator)
+        encoding_list.append(P)
+    return torch.tensor(np.array(encoding_list))
+
+
+
+

train.py

@@ -0,0 +1,12 @@
+from src.run import NLP_classification
+import wandb
+import yaml
+
+config_file = 'configs/config.yaml'
+config = yaml.load(open(config_file), Loader=yaml.FullLoader)
+
+trainer = NLP_classification(**config['model'])
+
+trainer.training(**config['train'], **config['wandb'])
+
+wandb.finish()