Init commit

1.0.py

@@ -0,0 +1,194 @@
+
+
+import pandas as pd
+import numpy as np
+import torch
+from transformers import RobertaTokenizer, RobertaForSequenceClassification
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from transformers import BertModel, RobertaModel
+from transformers import BertTokenizer, RobertaTokenizer
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from datetime import datetime
+from torch.autograd import Variable
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import Levenshtein
+import spacy
+import en_core_web_sm
+import torch.optim as optim
+from torch.distributions import Categorical
+from numpy import linalg as LA
+from transformers import AutoModelForMaskedLM
+from nltk.corpus import wordnet
+import torch.nn.functional as F
+import random
+from transformers import get_linear_schedule_with_warmup
+from sklearn.metrics import precision_recall_fscore_support
+from nltk.corpus import words as wal
+from sklearn.utils import resample
+
+
+# In[56]:
+
+
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        df1 = pd.read_csv(file_name)
+        df1 = df1.fillna("")
+        res = df1['X']
+        self.X_list = res.to_numpy()
+        self.y_list = df1['y'].to_numpy()
+    def __len__(self):
+        return len(self.X_list)
+    def __getitem__(self,idx):
+        mapi = []
+        mapi.append(self.X_list[idx])
+        mapi.append(self.y_list[idx])
+        return mapi
+
+
+# In[59]:
+
+
+class Step1_model(nn.Module):
+    def __init__(self, hidden_size=512):
+        super(Step1_model, self).__init__()
+        self.hidden_size = hidden_size
+        self.model = RobertaForSequenceClassification.from_pretrained("microsoft/graphcodebert-base", num_labels=6)
+        self.tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+        self.config = AutoConfig.from_pretrained("microsoft/graphcodebert-base")
+        for name, param in self.model.named_parameters():
+            param.requires_grad = True
+          
+
+    def forward(self, mapi):
+        X_init = mapi[0]
+        X_init = X_init.replace("[MASK]", " ".join([tokenizer.mask_token] * 1))
+        y = mapi[1]
+        print(y)
+        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        lb = ' '.join(nl).lower()
+        x = tokenizer.tokenize(lb)
+        nlab = len(x)
+        print(nlab)
+        tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+        input_id_chunki = tokens['input_ids'][0].split(510)
+        input_id_chunks = []
+        mask_chunks  = []
+        mask_chunki = tokens['attention_mask'][0].split(510)
+        for tensor in input_id_chunki:
+            input_id_chunks.append(tensor)
+        for tensor in mask_chunki:
+            mask_chunks.append(tensor)
+        xi = torch.full((1,), fill_value=101)
+        yi = torch.full((1,), fill_value=1)
+        zi = torch.full((1,), fill_value=102)
+        for r in range(len(input_id_chunks)):
+            input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+            input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+            mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+            mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+        di = torch.full((1,), fill_value=0)
+        for i in range(len(input_id_chunks)):
+            # get required padding length
+            pad_len = 512 - input_id_chunks[i].shape[0]
+            # check if tensor length satisfies required chunk size
+            if pad_len > 0:
+                # if padding length is more than 0, we must add padding
+                for p in range(pad_len):
+                    input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                    mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+        input_ids = torch.stack(input_id_chunks)
+        attention_mask = torch.stack(mask_chunks)
+        input_dict = {
+            'input_ids': input_ids.long(),
+            'attention_mask': attention_mask.int()
+        }
+        with torch.no_grad():
+            outputs = self.model(**input_dict)
+        last_hidden_state = outputs.logits.squeeze()
+        lhs_agg = []
+        if len(last_hidden_state) == 1:
+            lhs_agg.append(last_hidden_state)
+        else:
+            for p in range(len(last_hidden_state)):
+                lhs_agg.append(last_hidden_state[p])
+        lhs = lhs_agg[0]
+        for i in range(len(lhs_agg)):
+            if i == 0:
+                continue
+            lhs+=lhs_agg[i]
+        lhs/=len(lhs_agg)
+    #    print(lhs)
+        predicted_prob = torch.softmax(lhs, dim=0)
+        if nlab > 6:
+            nlab = 6
+        pll = -1*torch.log(predicted_prob[nlab-1])
+        return {'loss':pll}
+
+
+# In[60]:
+
+
+epoch_number = 0
+EPOCHS = 5
+run_int = 8
+tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+model = Step1_model()
+optimizer = optim.AdamW(model.parameters(), lr=2e-5)
+myDs=MyDataset('dat1.csv')
+train_loader=DataLoader(myDs,batch_size=1,shuffle=False)
+best_loss = torch.full((1,), fill_value=100000)
+
+
+# In[61]:
+
+
+flag = 0
+def train_one_epoch(transformer_model, dataset):
+    global flag
+    for batch in dataset:
+        p = 0
+        inputs = batch
+        optimizer.zero_grad()
+        for i in range(len(inputs[0])):
+            l = []
+            l.append(inputs[0][i])
+            l.append(inputs[1][i])
+            opi = transformer_model(l)
+            loss = opi['loss']
+        loss.backward()
+        optimizer.step()
+        if p % 1 == 0:
+            print('  batch loss: {}'.format(loss))
+    return loss
+
+
+# In[62]:
+
+
+for epoch in range(EPOCHS):
+    print('EPOCH {}:'.format(epoch_number + 1))
+
+    model.train(True)
+    avg_loss = train_one_epoch(model,train_loader)
+    model.train(False)
+    print('LOSS train {}'.format(avg_loss))
+    if avg_loss < best_loss:
+        best_loss = avg_loss
+    model_path = 'var_runs_class/model_{}_{}'.format(run_int, epoch_number)
+    torch.save(model.state_dict(), model_path)
+
+    epoch_number += 1
+
+
+# In[ ]:
+
+
+
+

2.2-all.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+from torch.nn.utils import clip_grad_norm_
+
+import pandas as pd
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.autograd import Variable
+import math
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import torch.optim as optim
+from tqdm import tqdm
+from transformers import AutoModelForMaskedLM
+import torch.nn.functional as F
+import random
+
+
+# In[2]:
+def freeze(model):
+    for name, param in model.named_parameters():
+        param.requires_grad = True
+        if name.startswith("model.roberta.encoder.layer.0"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.1"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.2"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.3"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.4"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.5"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.6"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.7"):
+            param.requires_grad = False
+   #     if name.startswith("model.roberta.encoder.layer.8"):
+  #          param.requires_grad = False
+#        if name.startswith("model.roberta.encoder.layer.9"):
+ #           param.requires_grad = False
+    return model
+
+maskis = []
+n_y = []
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        global maskis
+        global n_y
+        df = pd.read_csv(file_name)
+        df = df.fillna("")
+        self.inp_dicts = []
+        for r in range(df.shape[0]):
+            X_init = df['X'][r]
+            y = df['y'][r]
+            n_y.append(y)
+            nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+            lb = ' '.join(nl).lower()
+            x = tokenizer.tokenize(lb)
+            num_sub_tokens_label = len(x)
+            X_init = X_init.replace("[MASK]", " ".join([tokenizer.mask_token] * num_sub_tokens_label))
+            tokens = tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+            input_id_chunki = tokens['input_ids'][0].split(510)
+            input_id_chunks = []
+            mask_chunks  = []
+            mask_chunki = tokens['attention_mask'][0].split(510)
+            for tensor in input_id_chunki:
+                input_id_chunks.append(tensor)
+            for tensor in mask_chunki:
+                mask_chunks.append(tensor)
+            xi = torch.full((1,), fill_value=101)
+            yi = torch.full((1,), fill_value=1)
+            zi = torch.full((1,), fill_value=102)
+            for r in range(len(input_id_chunks)):
+                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+            di = torch.full((1,), fill_value=0)
+            for i in range(len(input_id_chunks)):
+                pad_len = 512 - input_id_chunks[i].shape[0]
+                if pad_len > 0:
+                    for p in range(pad_len):
+                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+            vb = torch.ones_like(input_id_chunks[0])
+            fg = torch.zeros_like(input_id_chunks[0])
+            maski = []
+            for l in range(len(input_id_chunks)):
+                masked_pos = []
+                for i in range(len(input_id_chunks[l])):
+                    if input_id_chunks[l][i] == tokenizer.mask_token_id: #103
+                        if i != 0 and input_id_chunks[l][i-1] == tokenizer.mask_token_id:
+                            continue
+                        masked_pos.append(i)
+                maski.append(masked_pos)
+            maskis.append(maski)
+            while (len(input_id_chunks)<250):
+                input_id_chunks.append(vb)
+                mask_chunks.append(fg)
+            input_ids = torch.stack(input_id_chunks)
+            attention_mask = torch.stack(mask_chunks)
+            input_dict = {
+                'input_ids': input_ids.long(),
+                'attention_mask': attention_mask.int()
+            }
+            self.inp_dicts.append(input_dict)
+            del input_dict
+            del input_ids
+            del attention_mask
+            del maski
+            del mask_chunks
+            del input_id_chunks
+            del di
+            del fg
+            del vb
+            del mask_chunki
+            del input_id_chunki
+            del X_init
+            del y
+            del tokens
+            del x
+            del lb
+            del nl
+        del df
+    def __len__(self):
+        return len(self.inp_dicts)
+    def __getitem__(self,idx):
+        return self.inp_dicts[idx]
+
+
+# In[3]:
+
+#device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+epoch_number = 0
+EPOCHS = 5
+run_int = 26
+tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+model = AutoModelForMaskedLM.from_pretrained("microsoft/graphcodebert-base")
+#model = model.half()
+#model = freeze(model)
+optimizer = Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=2e-5)
+myDs=MyDataset('dat1.csv') 
+train_loader=DataLoader(myDs,batch_size=1,shuffle=False)
+#model.half()
+#model = freeze(model)
+best_loss = torch.full((1,), fill_value=100000)
+#model = nn.DataParallel(model,device_ids=[0,1])
+#model.to(device)
+
+
+# In[5]:
+
+
+for epoch in range(EPOCHS):
+    loop = tqdm(train_loader, leave=True)
+    tot_loss = 0.0
+    cntr = 0
+#    nbtch = torch.tensor(0.0, requires_grad=True)
+    for batch in loop:
+        optimizer.zero_grad()
+        maxi = torch.tensor(0.0, requires_grad=True)
+        for i in range(len(batch['input_ids'])):            
+            cntr+=1
+            maski = maskis[cntr-1]
+            li = len(maski)
+            input_ids = batch['input_ids'][i][:li]
+            att_mask = batch['attention_mask'][i][:li]
+            y = n_y[cntr-1]
+            print("Ground truth:", y)
+            ty = tokenizer.encode(y)[1:-1]
+            num_sub_tokens_label = len(ty)
+#            input_ids, att_mask = input_ids.to(device),att_mask.to(device)
+            outputs = model(input_ids, attention_mask = att_mask)
+            last_hidden_state = outputs[0].squeeze()
+            l_o_l_sa = []
+            sum_state = []
+            for t in range(num_sub_tokens_label):
+                c = []
+                l_o_l_sa.append(c)
+            if len(maski) == 1:
+                masked_pos = maski[0]
+                for k in masked_pos:
+                    for t in range(num_sub_tokens_label):
+                        l_o_l_sa[t].append(last_hidden_state[k+t])
+            else:
+                for p in range(len(maski)):
+                    masked_pos = maski[p]
+                    for k in masked_pos:
+                        for t in range(num_sub_tokens_label):
+                            if (k+t) >= len(last_hidden_state[p]):
+                                l_o_l_sa[t].append(last_hidden_state[p+1][k+t-len(last_hidden_state[p])])
+                                continue
+                            l_o_l_sa[t].append(last_hidden_state[p][k+t])
+            for t in range(num_sub_tokens_label):
+                sum_state.append(l_o_l_sa[t][0])
+            for i in range(len(l_o_l_sa[0])):
+                if i == 0:
+                    continue
+                for t in range(num_sub_tokens_label):
+                    sum_state[t] = sum_state[t] + l_o_l_sa[t][i]
+            yip = len(l_o_l_sa[0])
+            qw = ""
+            val = torch.tensor(0.0, requires_grad=True)
+            for t in range(num_sub_tokens_label):
+                sum_state[t] /= yip
+                print("sum_state: ", sum_state[t])
+                print("yip: ", yip)
+                probs = F.softmax(sum_state[t], dim=0)
+                print("probs: ",probs)
+                print("idx: ", probs[ty[t]])
+                val = val - torch.log(probs[ty[t]])
+                idx = torch.topk(sum_state[t], k=5, dim=0)[1]
+                wor = [tokenizer.decode(i.item()).strip() for i in idx]
+                for kl in wor:
+                    if all(char.isalpha() for char in kl):
+                        qw+=kl.capitalize()
+                        break 
+            print("NTokens: ", num_sub_tokens_label)
+#            val = val / 5
+            val = val / num_sub_tokens_label
+            print("Val: ", val)
+            maxi = maxi + val
+            print("Prediction: ", qw)
+            print("*****")
+            for c in sum_state:
+                del c
+            del sum_state
+            for c in l_o_l_sa:
+                del c
+            del l_o_l_sa
+            del maski
+            del input_ids
+            del att_mask
+            del last_hidden_state
+            del qw
+
+        tot_loss +=maxi
+        maxi = maxi / len(batch['input_ids'])
+        maxi.backward()
+        optimizer.step()
+        if cntr%200 == 0:
+            checkpoint = {
+                'model_state_dict': model.state_dict(),
+                'optimizer_state_dict': optimizer.state_dict(),
+                'cntr': cntr  # Add any additional information you want>
+            }
+            model_path = 'var_runs/model_{}_{}_{}.pth'.format(run_>
+            torch.save(checkpoint, model_path)
+     #   nbtch = nbtch + maxi
+      #  if cntr % 4 == 0:
+     #       nbtch = nbtch / 4
+      #      nbtch.backward()
+      #      for name, param in model.named_parameters():
+       #         if param.grad is not None:
+       #             print(f'Parameter: {name}')
+       #             print(f'Gradient Norm: {param.grad.norm().item()}')
+               #     print(f'Gradient Values: {param.grad}')
+       #     max_grad_norm = 1.0  # You can adjust this value as needed
+       #     clip_grad_norm_(model.parameters(), max_grad_norm)
+       #     optimizer.step()
+        #    nbtch = 0.0
+       #     print(list(model.parameters())[0].grad)
+        loop.set_description(f'Epoch {epoch}')
+        loop.set_postfix(loss=maxi.item())
+    tot_loss/=len(myDs)
+    print(tot_loss)
+    if tot_loss < best_loss:
+        best_loss = tot_loss
+    model_path = 'var_runs/model_{}_{}'.format(run_int, epoch)
+    torch.save(model.state_dict(), model_path)
+
+
+# In[ ]:
+
+
+
+

2.2-iter.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import pandas as pd
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.autograd import Variable
+import math
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import torch.optim as optim
+from tqdm import tqdm
+from transformers import AutoModelForMaskedLM
+import torch.nn.functional as F
+import random
+
+
+# In[2]:
+
+def freeze(model):
+    for name, param in model.named_parameters():
+        param.requires_grad = True
+        if name.startswith("model.roberta.encoder.layer.0"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.1"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.2"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.3"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.4"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.5"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.6"):
+            param.requires_grad = False
+        if name.startswith("model.roberta.encoder.layer.7"):
+            param.requires_grad = False
+   #     if name.startswith("model.roberta.encoder.layer.8"):
+  #          param.requires_grad = False
+#        if name.startswith("model.roberta.encoder.layer.9"):
+ #           param.requires_grad = False
+    return model
+
+maskis = []
+n_y = []
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        global maskis
+        global n_y
+        df = pd.read_csv(file_name)
+        df = df.sample(frac=1)
+        df = df.fillna("")
+        self.inp_dicts = []
+        tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+        for r in range(df.shape[0]):
+            X_init = df['X'][r]
+            y = df['y'][r]
+            n_y.append(y)
+            nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+            lb = ' '.join(nl).lower()
+            x = tokenizer.tokenize(lb)
+            num_sub_tokens_label = len(x)
+            X_init = X_init.replace("[MASK]", " ".join([tokenizer.mask_token] * num_sub_tokens_label))
+            tokens = tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+            input_id_chunki = tokens['input_ids'][0].split(510)
+            input_id_chunks = []
+            mask_chunks  = []
+            mask_chunki = tokens['attention_mask'][0].split(510)
+            for tensor in input_id_chunki:
+                input_id_chunks.append(tensor)
+            for tensor in mask_chunki:
+                mask_chunks.append(tensor)
+            xi = torch.full((1,), fill_value=101)
+            yi = torch.full((1,), fill_value=1)
+            zi = torch.full((1,), fill_value=102)
+            for r in range(len(input_id_chunks)):
+                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+            di = torch.full((1,), fill_value=0)
+            for i in range(len(input_id_chunks)):
+                pad_len = 512 - input_id_chunks[i].shape[0]
+                if pad_len > 0:
+                    for p in range(pad_len):
+                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+            vb = torch.ones_like(input_id_chunks[0])
+            fg = torch.zeros_like(input_id_chunks[0])
+            maski = []
+            for l in range(len(input_id_chunks)):
+                masked_pos = []
+                for i in range(len(input_id_chunks[l])):
+                    if input_id_chunks[l][i] == tokenizer.mask_token_id: #103
+                        if i != 0 and input_id_chunks[l][i-1] == tokenizer.mask_token_id:
+                            continue
+                        masked_pos.append(i)
+                maski.append(masked_pos)
+            maskis.append(maski)
+            while (len(input_id_chunks)<250):
+                input_id_chunks.append(vb)
+                mask_chunks.append(fg)
+            input_ids = torch.stack(input_id_chunks)
+            attention_mask = torch.stack(mask_chunks)
+            input_dict = {
+                'input_ids': input_ids.long(),
+                'attention_mask': attention_mask.int()
+            }
+            self.inp_dicts.append(input_dict)
+            del input_dict
+            del input_ids
+            del attention_mask
+            del maski
+            del mask_chunks
+            del input_id_chunks
+            del di
+            del fg
+            del vb
+            del mask_chunki
+            del input_id_chunki
+            del X_init
+            del y
+            del tokens
+            del x
+            del lb
+            del nl
+        del df
+    def __len__(self):
+        return len(self.inp_dicts)
+    def __getitem__(self,idx):
+        return self.inp_dicts[idx]
+
+
+# In[3]:
+
+
+def my_func():
+    global maskis
+    global n_y
+    epoch_number = 0
+    EPOCHS = 5
+    run_int = 26
+    tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+    model = AutoModelForMaskedLM.from_pretrained("microsoft/graphcodebert-base")
+#    model = model.half()
+#    model = freeze(model)
+    optimizer = Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=2e-5)
+    myDs=MyDataset('dat1.csv') 
+    train_loader=DataLoader(myDs,batch_size=1,shuffle=False)
+    best_loss = torch.full((1,), fill_value=100000)
+    for epoch in range(EPOCHS):
+        loop = tqdm(train_loader, leave=True)
+        tot_loss = 0.0
+        cntr = 0
+        for batch in loop:
+            try:
+                optimizer.zero_grad()
+                maxi = torch.tensor(0.0, requires_grad=True)
+                for i in range(len(batch['input_ids'])):            
+                    cntr+=1
+                    maski = maskis[cntr-1]
+                    li = len(maski)
+                    input_ids = batch['input_ids'][i][:li]
+                    att_mask = batch['attention_mask'][i][:li]
+                    y = n_y[cntr-1]
+                    print("Ground truth:", y)
+                    ty = tokenizer.encode(y)[1:-1]
+                    num_sub_tokens_label = len(ty)
+                    qw = ""
+                    val = torch.tensor(0.0, requires_grad=True)
+                    for m in range(num_sub_tokens_label):
+                        outputs = model(input_ids, attention_mask = att_mask)
+                        last_hidden_state = outputs[0].squeeze()
+                        l_o_l_sa = []
+                        sum_state = []
+                        if len(maski) == 1:
+                            masked_pos = maski[0]
+                            for k in masked_pos:
+                                l_o_l_sa.append(last_hidden_state[k+m])
+                        else:
+                            for p in range(len(maski)):
+                                masked_pos = maski[p]
+                                for k in masked_pos:
+                                    if (k+m) >= len(last_hidden_state[p]):
+                                        l_o_l_sa.append(last_hidden_state[p+1][k+m-len(last_hidden_state[p])])
+                                        continue
+                                    l_o_l_sa.append(last_hidden_state[p][k+m])
+                        sum_state = l_o_l_sa[0]
+                        for i in range(len(l_o_l_sa)):
+                            if i == 0:
+                                continue
+                            sum_state = sum_state + l_o_l_sa[i]
+                        yip = len(l_o_l_sa)
+                        sum_state = sum_state / yip
+                        probs = F.softmax(sum_state, dim=0)
+                        val = val - torch.log(probs[ty[m]])
+                        idx = torch.topk(sum_state, k=5, dim=0)[1]
+                        wor= [tokenizer.decode(i.item()).strip() for i in idx]  
+                        for kl in wor:
+                            if all(char.isalpha() for char in kl):
+                                qw+=kl.capitalize()
+                                break 
+                        des = input_ids.clone()
+                        if len(maski) == 1:
+                            masked_pos = maski[0]
+                            for k in masked_pos:
+                                des[k+m] = idx[0]
+                        else:
+                            for p in range(len(maski)):
+                                masked_pos = maski[p]
+                                for k in masked_pos:
+                                    if (k+m) >= len(des[p]):
+                                        des[p+1][k+m-len(des[p])] = idx[0]
+                                        continue
+                                    des[p][k+m] = idx[0]
+                        del input_ids
+                        input_ids = des
+                        for c in sum_state:
+                            del c
+                        del sum_state
+                        for c in l_o_l_sa:
+                            del c
+                        del l_o_l_sa
+                        del last_hidden_state
+                    val = val / num_sub_tokens_label
+                    maxi = maxi + val
+                    print("Prediction: ", qw)
+                    print("*****")
+                    del maski
+                    del input_ids
+                    del att_mask
+                    del qw
+                tot_loss +=maxi
+                maxi = maxi / len(batch['input_ids'])
+                maxi.backward()
+                optimizer.step()
+                if cntr%200 == 0:
+                    checkpoint = {
+                        'model_state_dict': model.state_dict(),
+                        'optimizer_state_dict': optimizer.state_dict(),
+                        'cntr': cntr  # Add any additional information you want>
+                    }
+                    model_path = 'var_runs_iter/model_{}_{}_{}.pth'.format(run_>
+                    torch.save(checkpoint,model_path)
+#                print(list(model.parameters())[0].grad)
+                loop.set_description(f'Epoch {epoch}')
+                loop.set_postfix(loss=maxi.item())
+            except:
+                continue
+        tot_loss/=len(myDs)
+        print(tot_loss)
+        if tot_loss < best_loss:
+            best_loss = tot_loss
+        model_path = 'var_runs_iter/model_{}_{}'.format(run_int, epoch)
+        torch.save(model.state_dict(), model_path)
+
+
+# In[ ]:
+
+
+if __name__ == "__main__":
+    my_func()
+
+
+# In[ ]:
+
+
+
+

4.0.ipynb

@@ -0,0 +1,154 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7a478c86",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-07-31T01:55:12.877222Z",
+     "start_time": "2023-07-31T01:55:12.874203Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "61571547",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-07-31T01:57:23.295679Z",
+     "start_time": "2023-07-31T01:57:23.292514Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "t = 2314"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "5d7eda84",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-07-31T01:56:12.033568Z",
+     "start_time": "2023-07-31T01:56:12.025127Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def download_java_files(url, path):\n",
+    "    global t\n",
+    "    # Send GET request to retrieve the folder contents\n",
+    "    response = requests.get(url, params={\"ref\": branch})\n",
+    "    if response.status_code == 200:\n",
+    "        # Parse the response JSON\n",
+    "        contents = response.json()\n",
+    "\n",
+    "        for item in contents:\n",
+    "            if item[\"type\"] == \"file\" and item[\"name\"].endswith(\".java\"):\n",
+    "                # Download Java file\n",
+    "                download_url = item[\"download_url\"]\n",
+    "                file_name = \"train_\"+str(t)\n",
+    "                output_file_path = f\"Desktop/MITACS/Dataset/inp-txt/{file_name}\"\n",
+    "\n",
+    "                # Send GET request to download the file content\n",
+    "                file_content = requests.get(download_url).text\n",
+    "\n",
+    "                # Save the file to the output directory\n",
+    "                with open(output_file_path, \"w\") as output_file:\n",
+    "                    output_file.write(file_content)\n",
+    "\n",
+    "                print(f\"Downloaded: {file_name}\")\n",
+    "                t+=1\n",
+    "            elif item[\"type\"] == \"dir\":\n",
+    "                # Recursively navigate into subfolders\n",
+    "                subfolder_url = item[\"url\"]\n",
+    "                subfolder_path = f\"{path}/{item['name']}\"\n",
+    "                download_java_files(subfolder_url, subfolder_path)\n",
+    "    else:\n",
+    "        print(\"Failed to retrieve folder contents.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "dafd2b43",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-07-31T01:59:48.822445Z",
+     "start_time": "2023-07-31T01:59:48.667420Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Failed to retrieve folder contents.\n",
+      "2314\n"
+     ]
+    }
+   ],
+   "source": [
+    "# GitHub repository details\n",
+    "repo = \"leakcanary\"\n",
+    "owner = \"square\"\n",
+    "branch = \"main\"  # Specify the branch you want to download from\n",
+    "output_directory = \"DesktopDataset/inp-txt\"  # Specify the directory to save the downloaded files\n",
+    "# API endpoint to get the list of files in the repository\n",
+    "url = f\"https://api.github.com/repos/{owner}/{repo}/contents\"\n",
+    "download_java_files(url, output_directory)\n",
+    "print(t)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f51e193d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10 (tensorflown)",
+   "language": "python",
+   "name": "tensorflown"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

README.md

@@ -0,0 +1,13 @@
+# Identifier-Renaming
+Generating higher quality identifier names by using context and following conventions with RLHF <br>
+0.0 is used to generate the csv dataset from the .java files after 4.0 <br>
+1.0 is code for the classifier to predict the number of mask tokens to insert in for the variable name <br>
+2.2-all is used for finetuning the GraphCodeBert model on variable names <br>
+4.0 is used to create the dataset. Repositories are cloned and this code file iterates over all the files and preprocesses them for creation of dataset.<br>
+procTest.ipynb processes the text generated while evaluating the trained model and generates relevant graphs<br>
+class_eval.py is used to evaluate the performance of the classifer <br>
+identifier_scoring.py uses two non-fine-tuned models GraphCodeBERT and CodeBERT for the metric<br>
+stat_sampling.py evaluates the use of random sampling technique to predict number of mask tokens<br>
+model_eval.py is the code to evaluate the trained model<br>
+model_test.csv is the subset of data used to evaluate the model<br>
+test.csv is the dataset used for the evaluation of the readability metric and the fine-tuned model <br>

class_eval.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[73]:
+
+
+import pandas as pd
+import numpy as np
+import torch
+from transformers import RobertaTokenizer, RobertaForSequenceClassification
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from transformers import BertModel, RobertaModel
+from transformers import BertTokenizer, RobertaTokenizer
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from datetime import datetime
+from torch.autograd import Variable
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import Levenshtein
+import spacy
+import en_core_web_sm
+import torch.optim as optim
+from torch.distributions import Categorical
+from numpy import linalg as LA
+from transformers import AutoModelForMaskedLM
+from nltk.corpus import wordnet
+import torch.nn.functional as F
+import random
+from transformers import get_linear_schedule_with_warmup
+from sklearn.metrics import precision_recall_fscore_support
+from nltk.corpus import words as wal
+from sklearn.utils import resample
+
+
+# In[56]:
+
+
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        df1 = pd.read_csv(file_name)
+        df1 = df1[230000:]
+        df1 = df1.fillna("")
+        res = df1['X']
+        self.X_list = res.to_numpy()
+        self.y_list = df1['y'].to_numpy()
+    def __len__(self):
+        return len(self.X_list)
+    def __getitem__(self,idx):
+        mapi = []
+        mapi.append(self.X_list[idx])
+        mapi.append(self.y_list[idx])
+        return mapi
+
+
+# In[59]:
+
+
+class Step1_model(nn.Module):
+    def __init__(self, hidden_size=512):
+        super(Step1_model, self).__init__()
+        self.hidden_size = hidden_size
+        self.model = RobertaForSequenceClassification.from_pretrained("microsoft/codebert-base", num_labels=6)
+        self.tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base")
+        self.config = AutoConfig.from_pretrained("microsoft/codebert-base")
+
+    def forward(self, mapi):
+        X_init = mapi[0]
+        X_init = X_init.replace("[MASK]", " ".join([tokenizer.mask_token] * 1))
+        y = mapi[1]
+        print(y)
+        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        lb = ' '.join(nl).lower()
+        x = tokenizer.tokenize(lb)
+        nlab = len(x)
+        print(nlab)
+        tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+        input_id_chunki = tokens['input_ids'][0].split(510)
+        input_id_chunks = []
+        mask_chunks  = []
+        mask_chunki = tokens['attention_mask'][0].split(510)
+        for tensor in input_id_chunki:
+            input_id_chunks.append(tensor)
+        for tensor in mask_chunki:
+            mask_chunks.append(tensor)
+        xi = torch.full((1,), fill_value=101)
+        yi = torch.full((1,), fill_value=1)
+        zi = torch.full((1,), fill_value=102)
+        for r in range(len(input_id_chunks)):
+            input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+            input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+            mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+            mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+        di = torch.full((1,), fill_value=0)
+        for i in range(len(input_id_chunks)):
+            pad_len = 512 - input_id_chunks[i].shape[0]
+            if pad_len > 0:
+                for p in range(pad_len):
+                    input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                    mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+        input_ids = torch.stack(input_id_chunks)
+        attention_mask = torch.stack(mask_chunks)
+        input_dict = {
+            'input_ids': input_ids.long(),
+            'attention_mask': attention_mask.int()
+        }
+        with torch.no_grad():
+            outputs = self.model(**input_dict)
+        last_hidden_state = outputs.logits.squeeze()
+        lhs_agg = []
+        if len(last_hidden_state) == 1:
+            lhs_agg.append(last_hidden_state)
+        else:
+            for p in range(len(last_hidden_state)):
+                lhs_agg.append(last_hidden_state[p])
+        lhs = lhs_agg[0]
+        for i in range(len(lhs_agg)):
+            if i == 0:
+                continue
+            lhs+=lhs_agg[i]
+        lhs/=len(lhs_agg)
+        print(lhs)
+        predicted_prob = torch.softmax(lhs, dim=0)
+        if nlab > 6:
+            nlab = 6
+        pll = -1*torch.log(predicted_prob[nlab-1])
+        
+        pred = torch.argmax(predicted_prob).item()
+        pred+=1
+        print(pred)
+        predicted = torch.tensor([pred], dtype = float) 
+        if pred == nlab:
+            l2 = 0
+        else:
+            l2 = 1
+        actual = torch.tensor([nlab], dtype = float) 
+        l1 = Variable(torch.tensor([(actual-predicted)**2],dtype=float),requires_grad = True)
+        return {'loss1':l1, 'loss2':l2}
+
+
+# In[60]:
+
+
+epoch_number = 0
+EPOCHS = 5
+run_int = 0
+tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base")
+model = Step1_model()
+myDs=MyDataset('dat_test.csv')
+train_loader=DataLoader(myDs,batch_size=2,shuffle=True)
+best_loss = torch.full((1,), fill_value=100000)
+
+
+# In[61]:
+
+
+flag = 0
+def train_one_epoch(transformer_model, dataset):
+    global flag
+    tot_loss1 = 0.0
+    tot_loss2 = 0.0
+    cnt = 0
+    for batch in dataset:
+        p = 0
+        inputs = batch
+        for i in range(len(inputs[0])):
+            cnt += 1
+            l = []
+            l.append(inputs[0][i])
+            l.append(inputs[1][i])
+            opi = transformer_model(l)
+            loss1 = opi['loss1']
+            loss2 = opi['loss2']
+            tot_loss1 += loss1
+            tot_loss2 += loss2
+
+    tot_loss1/=cnt
+    tot_loss2/=cnt
+    print('MSE: ')
+    print(tot_loss1)
+    print('Acc: ',tot_loss2)
+    return {'tot loss1': tot_loss1,'tot_loss2':tot_loss2}
+
+
+# In[62]:
+
+model.eval()
+avg_loss = train_one_epoch(model,train_loader)
+
+
+
+
+# In[ ]:
+
+
+
+

id_score.ipynb

@@ -0,0 +1,725 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "aba235f2",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:12:18.439224Z",
+     "start_time": "2023-09-27T18:12:12.646006Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch import nn\n",
+    "import torch.nn.functional as F\n",
+    "from torch.nn import init, MarginRankingLoss\n",
+    "from transformers import BertModel, RobertaModel\n",
+    "from transformers import BertTokenizer, RobertaTokenizer\n",
+    "from torch.optim import Adam\n",
+    "from distutils.version import LooseVersion\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from torch.utils.tensorboard import SummaryWriter\n",
+    "from datetime import datetime\n",
+    "from torch.autograd import Variable\n",
+    "from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer\n",
+    "import torch.optim as optim\n",
+    "from torch.distributions import Categorical\n",
+    "import random\n",
+    "from transformers import AutoModelForMaskedLM, BertForMaskedLM, AdamW\n",
+    "from transformers import BertTokenizer\n",
+    "from tqdm import tqdm\n",
+    "import matplotlib.pyplot as plt\n",
+    "from transformers import XLMRobertaTokenizer\n",
+    "import os\n",
+    "import csv\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "import nltk\n",
+    "from collections import defaultdict\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk import pos_tag\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "import math\n",
+    "from nltk.corpus import words\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "import random\n",
+    "import re\n",
+    "import random"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ddeeea22",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:12:18.442893Z",
+     "start_time": "2023-09-27T18:12:18.440610Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "class MyDataset(Dataset):\n",
+    "    def __init__(self,file_name):\n",
+    "        df1 = pd.read_csv(file_name)\n",
+    "        df1 = df1[200:300]\n",
+    "        df1 = df1.fillna(\"\")\n",
+    "        res = df1['X'].to_numpy()\n",
+    "        self.X_list = res\n",
+    "        self.y_list = df1['y'].to_numpy()\n",
+    "    def __len__(self):\n",
+    "        return len(self.X_list)\n",
+    "    def __getitem__(self,idx):\n",
+    "        mapi = []\n",
+    "        mapi.append(self.X_list[idx])\n",
+    "        mapi.append(self.y_list[idx])\n",
+    "        return mapi"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "dd2fe8b9",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:12:18.466279Z",
+     "start_time": "2023-09-27T18:12:18.443804Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "class Step1_model(nn.Module):\n",
+    "    def __init__(self, hidden_size=512):\n",
+    "#         global old_inp\n",
+    "#         global old_mhs\n",
+    "#         self.oi = old_inp\n",
+    "#         self.old_mhs = old_mhs\n",
+    "        super(Step1_model, self).__init__()\n",
+    "        self.hidden_size = hidden_size\n",
+    "#         self.model = AutoModel.from_pretrained(\"roberta-base\")\n",
+    "#         self.tokenizer = AutoTokenizer.from_pretrained(\"roberta-base\")\n",
+    "#         self.config = AutoConfig.from_pretrained(\"roberta-base\")\n",
+    "        self.model = AutoModelForMaskedLM.from_pretrained('microsoft/graphcodebert-base')\n",
+    "        self.tokenizer = AutoTokenizer.from_pretrained(\"microsoft/graphcodebert-base\")\n",
+    "        self.config = AutoConfig.from_pretrained(\"microsoft/graphcodebert-base\")\n",
+    "        self.linear_layer = nn.Linear(self.model.config.vocab_size, self.model.config.vocab_size)\n",
+    "\n",
+    "#         self.model = AutoModelForMaskedLM.from_pretrained('bert-base-cased')\n",
+    "#         self.tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
+    "#         self.config = AutoConfig.from_pretrained(\"bert-base-cased\")\n",
+    "        for param in self.model.base_model.parameters():\n",
+    "            param.requires_grad = True\n",
+    "    def foo (self,data):\n",
+    "        result = []\n",
+    "        if type(data) == tuple:\n",
+    "            return data[1]\n",
+    "        if type(data) == list:\n",
+    "            for inner in data:\n",
+    "                result.append(foo(inner))\n",
+    "        res = []\n",
+    "        for a in result[0]:\n",
+    "            res.append(a[:2])\n",
+    "        return res\n",
+    "    def loss_func1(self, word, y):\n",
+    "        if word =='NA':\n",
+    "            return torch.full((1,), fill_value=100)\n",
+    "        try:\n",
+    "            pred_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', word)\n",
+    "            target_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', y)\n",
+    "            pred_tag = self.foo(nltk.pos_tag(pred_list))\n",
+    "            target_tag = self.foo(nltk.pos_tag(target_list))\n",
+    "            str1 = ' '.join(pred_tag)  # Convert lists to strings\n",
+    "            str2 = ' '.join(target_tag)\n",
+    "            distance = Levenshtein.distance(str1, str2)\n",
+    "            dist = torch.Tensor([distance])\n",
+    "        except:\n",
+    "            dist = torch.Tensor([2*len(target_list)])\n",
+    "        return dist\n",
+    "    def loss_func2(self, word, y):\n",
+    "        if word =='NA':\n",
+    "            return  torch.full((1,), fill_value=100)\n",
+    "        nlp = en_core_web_sm.load()\n",
+    "        pred_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', word)\n",
+    "        target_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', y)\n",
+    "        try:\n",
+    "            str1 = ' '.join(pred_list)  # Convert lists to strings\n",
+    "            str2 = ' '.join(target_list)\n",
+    "            tokens1 = nlp(str1)\n",
+    "            tokens2 = nlp(str2)\n",
+    "            # Calculate the average word embedding for each string\n",
+    "            embedding1 = sum(token.vector for token in tokens1) / len(tokens1)\n",
+    "            embedding2 = sum(token.vector for token in tokens2) / len(tokens2)\n",
+    "            # Calculate the cosine similarity between the embeddings\n",
+    "            w1= LA.norm(embedding1)\n",
+    "            w2= LA.norm(embedding2)\n",
+    "            distance = 1 - (embedding1.dot(embedding2) / (w1 * w2))\n",
+    "            dist = torch.Tensor([distance])\n",
+    "        except:\n",
+    "            dist = torch.Tensor([1])\n",
+    "        return dist\n",
+    "    def forward(self, mapi):\n",
+    "        global variable_names\n",
+    "        global base_model\n",
+    "        global tot_pll\n",
+    "        global base_tot_pll\n",
+    "        X_init1 = mapi[0]\n",
+    "        X_init = mapi[0]\n",
+    "        y = mapi[1]\n",
+    "        print(y)\n",
+    "        y_tok = self.tokenizer.encode(y)[1:-1]\n",
+    "        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', y)\n",
+    "        lb = ' '.join(nl).lower()\n",
+    "        x = self.tokenizer.tokenize(lb)\n",
+    "        num_sub_tokens_label = len(x)\n",
+    "        X_init = X_init.replace(\"[MASK]\", \" \".join([self.tokenizer.mask_token] * num_sub_tokens_label))\n",
+    "        sent_pll = 0.0\n",
+    "        base_sent_pll = 0.0\n",
+    "        for m in range(num_sub_tokens_label):\n",
+    "            print(m)\n",
+    "            tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')\n",
+    "            input_id_chunki = tokens['input_ids'][0].split(510)\n",
+    "            input_id_chunks = []\n",
+    "            mask_chunks  = []\n",
+    "            mask_chunki = tokens['attention_mask'][0].split(510)\n",
+    "            for tensor in input_id_chunki:\n",
+    "                input_id_chunks.append(tensor)\n",
+    "            for tensor in mask_chunki:\n",
+    "                mask_chunks.append(tensor)\n",
+    "            xi = torch.full((1,), fill_value=101)\n",
+    "            yi = torch.full((1,), fill_value=1)\n",
+    "            zi = torch.full((1,), fill_value=102)\n",
+    "            for r in range(len(input_id_chunks)):\n",
+    "                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)\n",
+    "                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)\n",
+    "                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)\n",
+    "                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)\n",
+    "            di = torch.full((1,), fill_value=0)\n",
+    "            for i in range(len(input_id_chunks)):\n",
+    "                pad_len = 512 - input_id_chunks[i].shape[0]\n",
+    "                if pad_len > 0:\n",
+    "                    for p in range(pad_len):\n",
+    "                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)\n",
+    "                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)\n",
+    "            input_ids = torch.stack(input_id_chunks)\n",
+    "            attention_mask = torch.stack(mask_chunks)\n",
+    "            input_dict = {\n",
+    "                'input_ids': input_ids.long(),\n",
+    "                'attention_mask': attention_mask.int()\n",
+    "            }\n",
+    "            maski = []\n",
+    "            u = 0\n",
+    "            ad = 0\n",
+    "            for l in range(len(input_dict['input_ids'])):\n",
+    "                masked_pos = []\n",
+    "                for i in range(len(input_dict['input_ids'][l])):\n",
+    "                    if input_dict['input_ids'][l][i] == 50264: #103\n",
+    "                        u+=1\n",
+    "                        if i != 0 and input_dict['input_ids'][l][i-1] == 50264:\n",
+    "                            continue\n",
+    "                        masked_pos.append(i)\n",
+    "                        ad+=1\n",
+    "                maski.append(masked_pos)\n",
+    "            print('number of mask tok',u)\n",
+    "            print('number of seq', ad)\n",
+    "            with torch.no_grad():\n",
+    "                output = self.model(**input_dict)\n",
+    "                base_output = base_model(**input_dict)\n",
+    "            last_hidden_state = output[0].squeeze()\n",
+    "            base_last_hidden_state = base_output[0].squeeze()\n",
+    "            l_o_l_sa = []\n",
+    "            base_l_o_l_sa = []\n",
+    "            if len(maski) == 1:\n",
+    "                masked_pos = maski[0]\n",
+    "                for k in masked_pos:\n",
+    "                    l_o_l_sa.append(last_hidden_state[k])\n",
+    "                    base_l_o_l_sa.append(base_last_hidden_state[k])\n",
+    "            else:\n",
+    "                for p in range(len(maski)):\n",
+    "                    masked_pos = maski[p]\n",
+    "                    for k in masked_pos:\n",
+    "                        l_o_l_sa.append(last_hidden_state[p][k])\n",
+    "                        base_l_o_l_sa.append(base_last_hidden_state[p][k])\n",
+    "            sum_state = l_o_l_sa[0]\n",
+    "            base_sum_state = base_l_o_l_sa[0]\n",
+    "            for i in range(len(l_o_l_sa)):\n",
+    "                if i == 0:\n",
+    "                    continue\n",
+    "                sum_state += l_o_l_sa[i]\n",
+    "                base_sum_state += base_l_o_l_sa[i]\n",
+    "            yip = len(l_o_l_sa)\n",
+    "            sum_state /= yip\n",
+    "            base_sum_state /= yip\n",
+    "            probs = F.softmax(sum_state, dim=0)\n",
+    "            base_probs = F.softmax(base_sum_state, dim=0)\n",
+    "            a_lab = y_tok[m]\n",
+    "            prob = probs[a_lab]\n",
+    "            base_prob = base_probs[a_lab]\n",
+    "            log_prob = -1*math.log(prob)\n",
+    "            base_log_prob = -1*math.log(base_prob)\n",
+    "            sent_pll+=log_prob\n",
+    "            base_sent_pll+=base_log_prob\n",
+    "            xl = X_init.split()\n",
+    "            xxl = []\n",
+    "            for p in range(len(xl)):\n",
+    "                if xl[p] == self.tokenizer.mask_token:\n",
+    "                    if p != 0 and xl[p-1] == self.tokenizer.mask_token:\n",
+    "                        xxl.append(xl[p])\n",
+    "                        continue\n",
+    "                    xxl.append(self.tokenizer.convert_ids_to_tokens(y_tok[m]))\n",
+    "                    continue\n",
+    "                xxl.append(xl[p])\n",
+    "            X_init = \" \".join(xxl)\n",
+    "        sent_pll/=num_sub_tokens_label\n",
+    "        base_sent_pll/=num_sub_tokens_label\n",
+    "        print(\"Sent PLL:\")\n",
+    "        print(sent_pll)\n",
+    "        print(\"Base Sent PLL:\")\n",
+    "        print(base_sent_pll)\n",
+    "        print(\"Net % difference:\")\n",
+    "        diff = (sent_pll-base_sent_pll)*100/base_sent_pll\n",
+    "        print(diff)\n",
+    "        tot_pll += sent_pll\n",
+    "        base_tot_pll+=base_sent_pll\n",
+    "        print()\n",
+    "        print()\n",
+    "        y = random.choice(variable_names)\n",
+    "        print(y)\n",
+    "        X_init = X_init1\n",
+    "        y_tok = self.tokenizer.encode(y)[1:-1]\n",
+    "        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\\d+', y)\n",
+    "        lb = ' '.join(nl).lower()\n",
+    "        x = self.tokenizer.tokenize(lb)\n",
+    "        num_sub_tokens_label = len(x)\n",
+    "        X_init = X_init.replace(\"[MASK]\", \" \".join([self.tokenizer.mask_token] * num_sub_tokens_label))\n",
+    "        sent_pll = 0.0\n",
+    "        base_sent_pll = 0.0\n",
+    "        for m in range(num_sub_tokens_label):\n",
+    "            print(m)\n",
+    "            tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')\n",
+    "            input_id_chunki = tokens['input_ids'][0].split(510)\n",
+    "            input_id_chunks = []\n",
+    "            mask_chunks  = []\n",
+    "            mask_chunki = tokens['attention_mask'][0].split(510)\n",
+    "            for tensor in input_id_chunki:\n",
+    "                input_id_chunks.append(tensor)\n",
+    "            for tensor in mask_chunki:\n",
+    "                mask_chunks.append(tensor)\n",
+    "            xi = torch.full((1,), fill_value=101)\n",
+    "            yi = torch.full((1,), fill_value=1)\n",
+    "            zi = torch.full((1,), fill_value=102)\n",
+    "            for r in range(len(input_id_chunks)):\n",
+    "                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)\n",
+    "                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)\n",
+    "                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)\n",
+    "                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)\n",
+    "            di = torch.full((1,), fill_value=0)\n",
+    "            for i in range(len(input_id_chunks)):\n",
+    "                pad_len = 512 - input_id_chunks[i].shape[0]\n",
+    "                if pad_len > 0:\n",
+    "                    for p in range(pad_len):\n",
+    "                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)\n",
+    "                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)\n",
+    "            input_ids = torch.stack(input_id_chunks)\n",
+    "            attention_mask = torch.stack(mask_chunks)\n",
+    "            input_dict = {\n",
+    "                'input_ids': input_ids.long(),\n",
+    "                'attention_mask': attention_mask.int()\n",
+    "            }\n",
+    "            maski = []\n",
+    "            u = 0\n",
+    "            ad = 0\n",
+    "            for l in range(len(input_dict['input_ids'])):\n",
+    "                masked_pos = []\n",
+    "                for i in range(len(input_dict['input_ids'][l])):\n",
+    "                    if input_dict['input_ids'][l][i] == 50264: #103\n",
+    "                        u+=1\n",
+    "                        if i != 0 and input_dict['input_ids'][l][i-1] == 50264:\n",
+    "                            continue\n",
+    "                        masked_pos.append(i)\n",
+    "                        ad+=1\n",
+    "                maski.append(masked_pos)\n",
+    "            print('number of mask tok',u)\n",
+    "            print('number of seq', ad)\n",
+    "            with torch.no_grad():\n",
+    "                output = self.model(**input_dict)\n",
+    "                base_output = base_model(**input_dict)\n",
+    "            last_hidden_state = output[0].squeeze()\n",
+    "            base_last_hidden_state = base_output[0].squeeze()\n",
+    "            l_o_l_sa = []\n",
+    "            base_l_o_l_sa = []\n",
+    "            if len(maski) == 1:\n",
+    "                masked_pos = maski[0]\n",
+    "                for k in masked_pos:\n",
+    "                    l_o_l_sa.append(last_hidden_state[k])\n",
+    "                    base_l_o_l_sa.append(base_last_hidden_state[k])\n",
+    "            else:\n",
+    "                for p in range(len(maski)):\n",
+    "                    masked_pos = maski[p]\n",
+    "                    for k in masked_pos:\n",
+    "                        l_o_l_sa.append(last_hidden_state[p][k])\n",
+    "                        base_l_o_l_sa.append(base_last_hidden_state[p][k])\n",
+    "            sum_state = l_o_l_sa[0]\n",
+    "            base_sum_state = base_l_o_l_sa[0]\n",
+    "            for i in range(len(l_o_l_sa)):\n",
+    "                if i == 0:\n",
+    "                    continue\n",
+    "                sum_state += l_o_l_sa[i]\n",
+    "                base_sum_state += base_l_o_l_sa[i]\n",
+    "            yip = len(l_o_l_sa)\n",
+    "            sum_state /= yip\n",
+    "            base_sum_state /= yip\n",
+    "            probs = F.softmax(sum_state, dim=0)\n",
+    "            base_probs = F.softmax(base_sum_state, dim=0)\n",
+    "            a_lab = y_tok[m]\n",
+    "            prob = probs[a_lab]\n",
+    "            base_prob = base_probs[a_lab]\n",
+    "            log_prob = -1*math.log(prob)\n",
+    "            base_log_prob = -1*math.log(base_prob)\n",
+    "            sent_pll+=log_prob\n",
+    "            base_sent_pll+=base_log_prob\n",
+    "            xl = X_init.split()\n",
+    "            xxl = []\n",
+    "            for p in range(len(xl)):\n",
+    "                if xl[p] == self.tokenizer.mask_token:\n",
+    "                    if p != 0 and xl[p-1] == self.tokenizer.mask_token:\n",
+    "                        xxl.append(xl[p])\n",
+    "                        continue\n",
+    "                    xxl.append(self.tokenizer.convert_ids_to_tokens(y_tok[m]))\n",
+    "                    continue\n",
+    "                xxl.append(xl[p])\n",
+    "            X_init = \" \".join(xxl)\n",
+    "        sent_pll/=num_sub_tokens_label\n",
+    "        base_sent_pll/=num_sub_tokens_label\n",
+    "        print(\"Sent PLL:\")\n",
+    "        print(sent_pll)\n",
+    "        print(\"Base Sent PLL:\")\n",
+    "        print(base_sent_pll)\n",
+    "        print(\"Net % difference:\")\n",
+    "        diff = (sent_pll-base_sent_pll)*100/base_sent_pll\n",
+    "        print(diff)\n",
+    "        print()\n",
+    "        print(\"******\")\n",
+    "        print()\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "bc788ca0",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:12:36.975722Z",
+     "start_time": "2023-09-27T18:12:18.467898Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "RobertaForMaskedLM(\n",
+       "  (roberta): RobertaModel(\n",
+       "    (embeddings): RobertaEmbeddings(\n",
+       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
+       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
+       "      (token_type_embeddings): Embedding(1, 768)\n",
+       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "      (dropout): Dropout(p=0.1, inplace=False)\n",
+       "    )\n",
+       "    (encoder): RobertaEncoder(\n",
+       "      (layer): ModuleList(\n",
+       "        (0-11): 12 x RobertaLayer(\n",
+       "          (attention): RobertaAttention(\n",
+       "            (self): RobertaSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): RobertaSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): RobertaIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (intermediate_act_fn): GELUActivation()\n",
+       "          )\n",
+       "          (output): RobertaOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (lm_head): RobertaLMHead(\n",
+       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "    (layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
+       "    (decoder): Linear(in_features=768, out_features=50265, bias=True)\n",
+       "  )\n",
+       ")"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/graphcodebert-base\")\n",
+    "model = Step1_model()\n",
+    "model.load_state_dict(torch.load('var_runs/model_98_3'))\n",
+    "base_model = AutoModelForMaskedLM.from_pretrained('microsoft/graphcodebert-base')\n",
+    "model.eval()\n",
+    "base_model.eval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f96328ce",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:15:14.635841Z",
+     "start_time": "2023-09-27T18:12:36.980040Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  0%|                                                    | 0/50 [00:00<?, ?it/s]"
+     ]
+    }
+   ],
+   "source": [
+    "myDs=MyDataset('dat.csv')\n",
+    "loader=DataLoader(myDs,batch_size=2,shuffle=True)\n",
+    "loop = tqdm(loader, leave=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "45333143",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2023-09-27T18:18:54.349042Z",
+     "start_time": "2023-09-27T18:17:34.313070Z"
+    },
+    "code_folding": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Token indices sequence length is longer than the specified maximum sequence length for this model (7050 > 512). Running this sequence through the model will result in indexing errors\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "stackBefore\n",
+      "\n",
+      "0\n",
+      "number of mask tok 16\n",
+      "number of seq 8\n",
+      "1\n",
+      "number of mask tok 8\n",
+      "number of seq 8\n",
+      "Sent PLL:\n",
+      "3.184066466322467\n",
+      "Base Sent PLL:\n",
+      "3.184066466322467\n",
+      "Net % difference:\n",
+      "0.0\n",
+      "\n",
+      "\n",
+      "distance\n",
+      "0\n",
+      "number of mask tok 8\n",
+      "number of seq 8\n",
+      "Sent PLL:\n",
+      "22.091890736746276\n",
+      "Base Sent PLL:\n",
+      "22.091890736746276\n",
+      "Net % difference:\n",
+      "0.0\n",
+      "\n",
+      "******\n",
+      "\n",
+      "records\n",
+      "\n",
+      "0\n",
+      "number of mask tok 4\n",
+      "number of seq 2\n",
+      "1\n",
+      "number of mask tok 2\n",
+      "number of seq 2\n",
+      "Sent PLL:\n",
+      "4.304520906089483\n",
+      "Base Sent PLL:\n",
+      "4.304520906089483\n",
+      "Net % difference:\n",
+      "0.0\n",
+      "\n",
+      "\n",
+      "valueB\n",
+      "0\n",
+      "number of mask tok 4\n",
+      "number of seq 2\n",
+      "1\n",
+      "number of mask tok 2\n",
+      "number of seq 2\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  2%|▊                                        | 1/50 [03:31<2:52:22, 211.08s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Sent PLL:\n",
+      "9.457522688945344\n",
+      "Base Sent PLL:\n",
+      "9.457522688945344\n",
+      "Net % difference:\n",
+      "0.0\n",
+      "\n",
+      "******\n",
+      "\n",
+      "stackEntry\n",
+      "\n",
+      "0\n",
+      "number of mask tok 30\n",
+      "number of seq 15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  2%|▊                                        | 1/50 [03:38<2:58:06, 218.09s/it]\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[6], line 18\u001b[0m\n\u001b[1;32m     16\u001b[0m         l\u001b[38;5;241m.\u001b[39mappend(inputs[\u001b[38;5;241m0\u001b[39m][i])\n\u001b[1;32m     17\u001b[0m         l\u001b[38;5;241m.\u001b[39mappend(inputs[\u001b[38;5;241m1\u001b[39m][i])\n\u001b[0;32m---> 18\u001b[0m         \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43ml\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;66;03m#         X_init1 = inputs[0][i]\u001b[39;00m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;66;03m#         X_init = inputs[0][i]\u001b[39;00m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;66;03m#         y = inputs[1][i]\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    262\u001b[0m \u001b[38;5;66;03m#     except:\u001b[39;00m\n\u001b[1;32m    263\u001b[0m \u001b[38;5;66;03m#         continue\u001b[39;00m\n\u001b[1;32m    264\u001b[0m tot_pll\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(myDs)\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "Cell \u001b[0;32mIn[3], line 152\u001b[0m, in \u001b[0;36mStep1_model.forward\u001b[0;34m(self, mapi)\u001b[0m\n\u001b[1;32m    150\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m    151\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minput_dict)\n\u001b[0;32m--> 152\u001b[0m     base_output \u001b[38;5;241m=\u001b[39m \u001b[43mbase_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minput_dict\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    153\u001b[0m last_hidden_state \u001b[38;5;241m=\u001b[39m output[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39msqueeze()\n\u001b[1;32m    154\u001b[0m base_last_hidden_state \u001b[38;5;241m=\u001b[39m base_output[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39msqueeze()\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:1082\u001b[0m, in \u001b[0;36mRobertaForMaskedLM.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1072\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   1073\u001b[0m \u001b[38;5;124;03mlabels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):\u001b[39;00m\n\u001b[1;32m   1074\u001b[0m \u001b[38;5;124;03m    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1078\u001b[0m \u001b[38;5;124;03m    Used to hide legacy arguments that have been deprecated.\u001b[39;00m\n\u001b[1;32m   1079\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   1080\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[0;32m-> 1082\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mroberta\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1083\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1084\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1085\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1086\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1087\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1088\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1089\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1090\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1091\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1092\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1093\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1094\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1095\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1096\u001b[0m prediction_scores \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(sequence_output)\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:844\u001b[0m, in \u001b[0;36mRobertaModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    835\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[1;32m    837\u001b[0m embedding_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings(\n\u001b[1;32m    838\u001b[0m     input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[1;32m    839\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    842\u001b[0m     past_key_values_length\u001b[38;5;241m=\u001b[39mpast_key_values_length,\n\u001b[1;32m    843\u001b[0m )\n\u001b[0;32m--> 844\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    845\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    846\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    847\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    848\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    849\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    850\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    851\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    852\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    853\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    854\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    855\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    856\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    857\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:529\u001b[0m, in \u001b[0;36mRobertaEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    520\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mcheckpoint\u001b[38;5;241m.\u001b[39mcheckpoint(\n\u001b[1;32m    521\u001b[0m         create_custom_forward(layer_module),\n\u001b[1;32m    522\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    526\u001b[0m         encoder_attention_mask,\n\u001b[1;32m    527\u001b[0m     )\n\u001b[1;32m    528\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 529\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    530\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    531\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    532\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    533\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    534\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    535\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    537\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    539\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    540\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:413\u001b[0m, in \u001b[0;36mRobertaLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    401\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    402\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    403\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    410\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m    411\u001b[0m     \u001b[38;5;66;03m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[39;00m\n\u001b[1;32m    412\u001b[0m     self_attn_past_key_value \u001b[38;5;241m=\u001b[39m past_key_value[:\u001b[38;5;241m2\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 413\u001b[0m     self_attention_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    414\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    415\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    416\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    417\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    418\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mself_attn_past_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    419\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    420\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m self_attention_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    422\u001b[0m     \u001b[38;5;66;03m# if decoder, the last output is tuple of self-attn cache\u001b[39;00m\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:340\u001b[0m, in \u001b[0;36mRobertaAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    330\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m    331\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    332\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    338\u001b[0m     output_attentions: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    339\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[0;32m--> 340\u001b[0m     self_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mself\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    341\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    342\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    343\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    344\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    345\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    346\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    347\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    348\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    349\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput(self_outputs[\u001b[38;5;241m0\u001b[39m], hidden_states)\n\u001b[1;32m    350\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (attention_output,) \u001b[38;5;241m+\u001b[39m self_outputs[\u001b[38;5;241m1\u001b[39m:]  \u001b[38;5;66;03m# add attentions if we output them\u001b[39;00m\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
+      "File \u001b[0;32m~/miniconda3/envs/tensorflown/lib/python3.10/site-packages/transformers/models/roberta/modeling_roberta.py:236\u001b[0m, in \u001b[0;36mRobertaSelfAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    233\u001b[0m     past_key_value \u001b[38;5;241m=\u001b[39m (key_layer, value_layer)\n\u001b[1;32m    235\u001b[0m \u001b[38;5;66;03m# Take the dot product between \"query\" and \"key\" to get the raw attention scores.\u001b[39;00m\n\u001b[0;32m--> 236\u001b[0m attention_scores \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmatmul\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_layer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey_layer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtranspose\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    238\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mposition_embedding_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrelative_key\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mposition_embedding_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrelative_key_query\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    239\u001b[0m     query_length, key_length \u001b[38;5;241m=\u001b[39m query_layer\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m2\u001b[39m], key_layer\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m2\u001b[39m]\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "tot_pll = 0.0\n",
+    "base_tot_pll = 0.0\n",
+    "variable_names = [\n",
+    "    'x', 'y', 'myVariable', 'dataPoint', 'randomNumber', 'userAge', 'resultValue', 'inputValue', 'tempValue', 'indexCounter', \n",
+    "    'itemPrice', 'userName', 'testScore', 'acceleration', 'productCount', 'errorMargin', 'piValue', 'sensorReading', \n",
+    "    'currentTemperature', 'velocityVector', 'variable1', 'variable2', 'valueA', 'valueB', 'counter', 'flag', 'total', \n",
+    "    'average', 'valueX', 'valueY', 'valueZ', 'price', 'quantity', 'name', 'age', 'score', 'weight', 'height', 'distance', \n",
+    "    'time', 'radius', 'width', 'length', 'temperature', 'pressure', 'humidity', 'voltage', 'current', 'resistance'\n",
+    "]\n",
+    "\n",
+    "for batch in loop:\n",
+    "    inputs = batch\n",
+    "    try:\n",
+    "        for i in range(len(inputs[0])):\n",
+    "            l = []\n",
+    "            l.append(inputs[0][i])\n",
+    "            l.append(inputs[1][i])\n",
+    "            model(l)\n",
+    "    except:\n",
+    "        continue\n",
+    "\n",
+    "tot_pll/=len(myDs)\n",
+    "print('Total PLL per sentence: ')\n",
+    "print(tot_pll)\n",
+    "base_tot_pll/=len(myDs)\n",
+    "print('Total Base PLL per sentence: ')\n",
+    "print(base_tot_pll)\n",
+    "print(\"Net % difference average:\")\n",
+    "tot_diff = (tot_pll-base_tot_pll)*100/base_tot_pll\n",
+    "print(tot_diff)\n",
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da79bcc2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10 (tensorflown)",
+   "language": "python",
+   "name": "tensorflown"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

identifier_scoring.py

@@ -0,0 +1,428 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import pandas as pd
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn import init, MarginRankingLoss
+from transformers import BertModel, RobertaModel
+from transformers import BertTokenizer, RobertaTokenizer
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from datetime import datetime
+from torch.autograd import Variable
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer
+import torch.optim as optim
+from torch.distributions import Categorical
+import random
+from transformers import AutoModelForMaskedLM, BertForMaskedLM, AdamW
+from transformers import BertTokenizer
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+from transformers import XLMRobertaTokenizer
+import os
+import csv
+from sklearn.model_selection import train_test_split
+import nltk
+from collections import defaultdict
+from nltk.tokenize import word_tokenize
+from nltk import pos_tag
+from nltk.tokenize import word_tokenize
+import math
+from nltk.corpus import words
+from sklearn.model_selection import train_test_split
+import random
+import re
+import random
+
+
+# In[2]:
+
+
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        df1 = pd.read_csv(file_name)
+        df1 = df1[200:300]
+        df1 = df1.fillna("")
+        res = df1['X'].to_numpy()
+        self.X_list = res
+        self.y_list = df1['y'].to_numpy()
+    def __len__(self):
+        return len(self.X_list)
+    def __getitem__(self,idx):
+        mapi = []
+        mapi.append(self.X_list[idx])
+        mapi.append(self.y_list[idx])
+        return mapi
+
+
+# In[3]:
+
+
+class Step1_model(nn.Module):
+    def __init__(self, hidden_size=512):
+        super(Step1_model, self).__init__()
+        self.hidden_size = hidden_size
+        self.model = AutoModelForMaskedLM.from_pretrained('microsoft/graphcodebert-base')
+        self.tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+        self.config = AutoConfig.from_pretrained("microsoft/graphcodebert-base")
+        self.linear_layer = nn.Linear(self.model.config.vocab_size, self.model.config.vocab_size)
+    def foo (self,data):
+        result = []
+        if type(data) == tuple:
+            return data[1]
+        if type(data) == list:
+            for inner in data:
+                result.append(foo(inner))
+        res = []
+        for a in result[0]:
+            res.append(a[:2])
+        return res
+    def loss_func1(self, word, y):
+        if word =='NA':
+            return torch.full((1,), fill_value=100)
+        try:
+            pred_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', word)
+            target_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+            pred_tag = self.foo(nltk.pos_tag(pred_list))
+            target_tag = self.foo(nltk.pos_tag(target_list))
+            str1 = ' '.join(pred_tag)  # Convert lists to strings
+            str2 = ' '.join(target_tag)
+            distance = Levenshtein.distance(str1, str2)
+            dist = torch.Tensor([distance])
+        except:
+            dist = torch.Tensor([2*len(target_list)])
+        return dist
+    def loss_func2(self, word, y):
+        if word =='NA':
+            return  torch.full((1,), fill_value=100)
+        nlp = en_core_web_sm.load()
+        pred_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', word)
+        target_list = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        try:
+            str1 = ' '.join(pred_list)  # Convert lists to strings
+            str2 = ' '.join(target_list)
+            tokens1 = nlp(str1)
+            tokens2 = nlp(str2)
+            embedding1 = sum(token.vector for token in tokens1) / len(tokens1)
+            embedding2 = sum(token.vector for token in tokens2) / len(tokens2)
+            w1= LA.norm(embedding1)
+            w2= LA.norm(embedding2)
+            distance = 1 - (embedding1.dot(embedding2) / (w1 * w2))
+            dist = torch.Tensor([distance])
+        except:
+            dist = torch.Tensor([1])
+        return dist
+    def forward(self, mapi):
+        global variable_names
+        global base_model
+        global tot_pll
+        global base_tot_pll
+        X_init1 = mapi[0]
+        X_init = mapi[0]
+        y = mapi[1]
+        print(y)
+        y_tok = self.tokenizer.encode(y)[1:-1]
+        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        lb = ' '.join(nl).lower()
+        x = self.tokenizer.tokenize(lb)
+        num_sub_tokens_label = len(x)
+        X_init = X_init.replace("[MASK]", " ".join([self.tokenizer.mask_token] * num_sub_tokens_label))
+        sent_pll = 0.0
+        base_sent_pll = 0.0
+        for m in range(num_sub_tokens_label):
+            print(m)
+            tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+            input_id_chunki = tokens['input_ids'][0].split(510)
+            input_id_chunks = []
+            mask_chunks  = []
+            mask_chunki = tokens['attention_mask'][0].split(510)
+            for tensor in input_id_chunki:
+                input_id_chunks.append(tensor)
+            for tensor in mask_chunki:
+                mask_chunks.append(tensor)
+            xi = torch.full((1,), fill_value=101)
+            yi = torch.full((1,), fill_value=1)
+            zi = torch.full((1,), fill_value=102)
+            for r in range(len(input_id_chunks)):
+                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+            di = torch.full((1,), fill_value=0)
+            for i in range(len(input_id_chunks)):
+                pad_len = 512 - input_id_chunks[i].shape[0]
+                if pad_len > 0:
+                    for p in range(pad_len):
+                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+            input_ids = torch.stack(input_id_chunks)
+            attention_mask = torch.stack(mask_chunks)
+            input_dict = {
+                'input_ids': input_ids.long(),
+                'attention_mask': attention_mask.int()
+            }
+            maski = []
+            u = 0
+            ad = 0
+            for l in range(len(input_dict['input_ids'])):
+                masked_pos = []
+                for i in range(len(input_dict['input_ids'][l])):
+                    if input_dict['input_ids'][l][i] == 50264: #103
+                        u+=1
+                        if i != 0 and input_dict['input_ids'][l][i-1] == 50264:
+                            continue
+                        masked_pos.append(i)
+                        ad+=1
+                maski.append(masked_pos)
+            print('number of mask tok',u)
+            print('number of seq', ad)
+            with torch.no_grad():
+                output = self.model(**input_dict)
+                base_output = base_model(**input_dict)
+            last_hidden_state = output[0].squeeze()
+            base_last_hidden_state = base_output[0].squeeze()
+            l_o_l_sa = []
+            base_l_o_l_sa = []
+            if len(maski) == 1:
+                masked_pos = maski[0]
+                for k in masked_pos:
+                    l_o_l_sa.append(last_hidden_state[k])
+                    base_l_o_l_sa.append(base_last_hidden_state[k])
+            else:
+                for p in range(len(maski)):
+                    masked_pos = maski[p]
+                    for k in masked_pos:
+                        l_o_l_sa.append(last_hidden_state[p][k])
+                        base_l_o_l_sa.append(base_last_hidden_state[p][k])
+            sum_state = l_o_l_sa[0]
+            base_sum_state = base_l_o_l_sa[0]
+            for i in range(len(l_o_l_sa)):
+                if i == 0:
+                    continue
+                sum_state += l_o_l_sa[i]
+                base_sum_state += base_l_o_l_sa[i]
+            yip = len(l_o_l_sa)
+            sum_state /= yip
+            base_sum_state /= yip
+            probs = F.softmax(sum_state, dim=0)
+            base_probs = F.softmax(base_sum_state, dim=0)
+            a_lab = y_tok[m]
+            prob = probs[a_lab]
+            base_prob = base_probs[a_lab]
+            log_prob = -1*math.log(prob)
+            base_log_prob = -1*math.log(base_prob)
+            sent_pll+=log_prob
+            base_sent_pll+=base_log_prob
+            xl = X_init.split()
+            xxl = []
+            for p in range(len(xl)):
+                if xl[p] == self.tokenizer.mask_token:
+                    if p != 0 and xl[p-1] == self.tokenizer.mask_token:
+                        xxl.append(xl[p])
+                        continue
+                    xxl.append(self.tokenizer.convert_ids_to_tokens(y_tok[m]))
+                    continue
+                xxl.append(xl[p])
+            X_init = " ".join(xxl)
+        sent_pll/=num_sub_tokens_label
+        base_sent_pll/=num_sub_tokens_label
+        print("Sent PLL:")
+        print(sent_pll)
+        print("Base Sent PLL:")
+        print(base_sent_pll)
+        print("Net % difference:")
+        diff = (sent_pll-base_sent_pll)*100/base_sent_pll
+        print(diff)
+        tot_pll += sent_pll
+        base_tot_pll+=base_sent_pll
+        print()
+        print()
+        y = random.choice(variable_names)
+        print(y)
+        X_init = X_init1
+        y_tok = self.tokenizer.encode(y)[1:-1]
+        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        lb = ' '.join(nl).lower()
+        x = self.tokenizer.tokenize(lb)
+        num_sub_tokens_label = len(x)
+        X_init = X_init.replace("[MASK]", " ".join([self.tokenizer.mask_token] * num_sub_tokens_label))
+        sent_pll = 0.0
+        base_sent_pll = 0.0
+        for m in range(num_sub_tokens_label):
+            print(m)
+            tokens = self.tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+            input_id_chunki = tokens['input_ids'][0].split(510)
+            input_id_chunks = []
+            mask_chunks  = []
+            mask_chunki = tokens['attention_mask'][0].split(510)
+            for tensor in input_id_chunki:
+                input_id_chunks.append(tensor)
+            for tensor in mask_chunki:
+                mask_chunks.append(tensor)
+            xi = torch.full((1,), fill_value=101)
+            yi = torch.full((1,), fill_value=1)
+            zi = torch.full((1,), fill_value=102)
+            for r in range(len(input_id_chunks)):
+                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+            di = torch.full((1,), fill_value=0)
+            for i in range(len(input_id_chunks)):
+                pad_len = 512 - input_id_chunks[i].shape[0]
+                if pad_len > 0:
+                    for p in range(pad_len):
+                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+            input_ids = torch.stack(input_id_chunks)
+            attention_mask = torch.stack(mask_chunks)
+            input_dict = {
+                'input_ids': input_ids.long(),
+                'attention_mask': attention_mask.int()
+            }
+            maski = []
+            u = 0
+            ad = 0
+            for l in range(len(input_dict['input_ids'])):
+                masked_pos = []
+                for i in range(len(input_dict['input_ids'][l])):
+                    if input_dict['input_ids'][l][i] == 50264: #103
+                        u+=1
+                        if i != 0 and input_dict['input_ids'][l][i-1] == 50264:
+                            continue
+                        masked_pos.append(i)
+                        ad+=1
+                maski.append(masked_pos)
+            print('number of mask tok',u)
+            print('number of seq', ad)
+            with torch.no_grad():
+                output = self.model(**input_dict)
+                base_output = base_model(**input_dict)
+            last_hidden_state = output[0].squeeze()
+            base_last_hidden_state = base_output[0].squeeze()
+            l_o_l_sa = []
+            base_l_o_l_sa = []
+            if len(maski) == 1:
+                masked_pos = maski[0]
+                for k in masked_pos:
+                    l_o_l_sa.append(last_hidden_state[k])
+                    base_l_o_l_sa.append(base_last_hidden_state[k])
+            else:
+                for p in range(len(maski)):
+                    masked_pos = maski[p]
+                    for k in masked_pos:
+                        l_o_l_sa.append(last_hidden_state[p][k])
+                        base_l_o_l_sa.append(base_last_hidden_state[p][k])
+            sum_state = l_o_l_sa[0]
+            base_sum_state = base_l_o_l_sa[0]
+            for i in range(len(l_o_l_sa)):
+                if i == 0:
+                    continue
+                sum_state += l_o_l_sa[i]
+                base_sum_state += base_l_o_l_sa[i]
+            yip = len(l_o_l_sa)
+            sum_state /= yip
+            base_sum_state /= yip
+            probs = F.softmax(sum_state, dim=0)
+            base_probs = F.softmax(base_sum_state, dim=0)
+            a_lab = y_tok[m]
+            prob = probs[a_lab]
+            base_prob = base_probs[a_lab]
+            log_prob = -1*math.log(prob)
+            base_log_prob = -1*math.log(base_prob)
+            sent_pll+=log_prob
+            base_sent_pll+=base_log_prob
+            xl = X_init.split()
+            xxl = []
+            for p in range(len(xl)):
+                if xl[p] == self.tokenizer.mask_token:
+                    if p != 0 and xl[p-1] == self.tokenizer.mask_token:
+                        xxl.append(xl[p])
+                        continue
+                    xxl.append(self.tokenizer.convert_ids_to_tokens(y_tok[m]))
+                    continue
+                xxl.append(xl[p])
+            X_init = " ".join(xxl)
+        sent_pll/=num_sub_tokens_label
+        base_sent_pll/=num_sub_tokens_label
+        print("Sent PLL:")
+        print(sent_pll)
+        print("Base Sent PLL:")
+        print(base_sent_pll)
+        print("Net % difference:")
+        diff = (sent_pll-base_sent_pll)*100/base_sent_pll
+        print(diff)
+        print()
+        print("******")
+        print()
+    
+
+
+# In[4]:
+
+
+tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+model = Step1_model()
+model.load_state_dict(torch.load('var_runs/model_98_3'))
+base_model = AutoModelForMaskedLM.from_pretrained('microsoft/graphcodebert-base')
+model.eval()
+base_model.eval()
+
+
+# In[5]:
+
+
+myDs=MyDataset('dat.csv')
+loader=DataLoader(myDs,batch_size=2,shuffle=True)
+loop = tqdm(loader, leave=True)
+
+
+# In[6]:
+
+
+tot_pll = 0.0
+base_tot_pll = 0.0
+variable_names = [
+    'x', 'y', 'myVariable', 'dataPoint', 'randomNumber', 'userAge', 'resultValue', 'inputValue', 'tempValue', 'indexCounter', 
+    'itemPrice', 'userName', 'testScore', 'acceleration', 'productCount', 'errorMargin', 'piValue', 'sensorReading', 
+    'currentTemperature', 'velocityVector', 'variable1', 'variable2', 'valueA', 'valueB', 'counter', 'flag', 'total', 
+    'average', 'valueX', 'valueY', 'valueZ', 'price', 'quantity', 'name', 'age', 'score', 'weight', 'height', 'distance', 
+    'time', 'radius', 'width', 'length', 'temperature', 'pressure', 'humidity', 'voltage', 'current', 'resistance'
+]
+
+for batch in loop:
+    inputs = batch
+    try:
+        for i in range(len(inputs[0])):
+            l = []
+            l.append(inputs[0][i])
+            l.append(inputs[1][i])
+            model(l)
+    except:
+        continue
+
+tot_pll/=len(myDs)
+print('Total PLL per sentence: ')
+print(tot_pll)
+base_tot_pll/=len(myDs)
+print('Total Base PLL per sentence: ')
+print(base_tot_pll)
+print("Net % difference average:")
+tot_diff = (tot_pll-base_tot_pll)*100/base_tot_pll
+print(tot_diff)
+  
+
+
+# In[ ]:
+
+
+
+

model_eval.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import pandas as pd
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.autograd import Variable
+import math
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import torch.optim as optim
+from tqdm import tqdm
+from transformers import AutoModelForMaskedLM
+import torch.nn.functional as F
+import random
+
+
+# In[2]:
+
+
+maskis = []
+n_y = []
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        global maskis
+        global n_y
+        df = pd.read_csv(file_name)
+        df = df.fillna("")
+        self.inp_dicts = []
+        for r in range(df.shape[0]):
+            X_init = df['X'][r]
+            y = df['y'][r]
+            n_y.append(y)
+            nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+            lb = ' '.join(nl).lower()
+            x = tokenizer.tokenize(lb)
+            num_sub_tokens_label = len(x)
+            X_init = X_init.replace("[MASK]", " ".join([tokenizer.mask_token] * num_sub_tokens_label))
+            tokens = tokenizer.encode_plus(X_init, add_special_tokens=False,return_tensors='pt')
+            input_id_chunki = tokens['input_ids'][0].split(510)
+            input_id_chunks = []
+            mask_chunks  = []
+            mask_chunki = tokens['attention_mask'][0].split(510)
+            for tensor in input_id_chunki:
+                input_id_chunks.append(tensor)
+            for tensor in mask_chunki:
+                mask_chunks.append(tensor)
+            xi = torch.full((1,), fill_value=101)
+            yi = torch.full((1,), fill_value=1)
+            zi = torch.full((1,), fill_value=102)
+            for r in range(len(input_id_chunks)):
+                input_id_chunks[r] = torch.cat([xi, input_id_chunks[r]],dim = -1)
+                input_id_chunks[r] = torch.cat([input_id_chunks[r],zi],dim=-1)
+                mask_chunks[r] = torch.cat([yi, mask_chunks[r]],dim=-1)
+                mask_chunks[r] = torch.cat([mask_chunks[r],yi],dim=-1)
+            di = torch.full((1,), fill_value=0)
+            for i in range(len(input_id_chunks)):
+                pad_len = 512 - input_id_chunks[i].shape[0]
+                if pad_len > 0:
+                    for p in range(pad_len):
+                        input_id_chunks[i] = torch.cat([input_id_chunks[i],di],dim=-1)
+                        mask_chunks[i] = torch.cat([mask_chunks[i],di],dim=-1)
+            vb = torch.ones_like(input_id_chunks[0])
+            fg = torch.zeros_like(input_id_chunks[0])
+            maski = []
+            for l in range(len(input_id_chunks)):
+                masked_pos = []
+                for i in range(len(input_id_chunks[l])):
+                    if input_id_chunks[l][i] == tokenizer.mask_token_id: #103
+                        if i != 0 and input_id_chunks[l][i-1] == tokenizer.mask_token_id:
+                            continue
+                        masked_pos.append(i)
+                maski.append(masked_pos)
+            maskis.append(maski)
+            while (len(input_id_chunks)<250):
+                input_id_chunks.append(vb)
+                mask_chunks.append(fg)
+            input_ids = torch.stack(input_id_chunks)
+            attention_mask = torch.stack(mask_chunks)
+            input_dict = {
+                'input_ids': input_ids.long(),
+                'attention_mask': attention_mask.int()
+            }
+            self.inp_dicts.append(input_dict)
+            del input_dict
+            del input_ids
+            del attention_mask
+            del maski
+            del mask_chunks
+            del input_id_chunks
+            del di
+            del fg
+            del vb
+            del mask_chunki
+            del input_id_chunki
+            del X_init
+            del y
+            del tokens
+            del x
+            del lb
+            del nl
+        del df
+    def __len__(self):
+        return len(self.inp_dicts)
+    def __getitem__(self,idx):
+        return self.inp_dicts[idx]
+
+
+# In[3]:
+
+
+tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+model = AutoModelForMaskedLM.from_pretrained("microsoft/graphcodebert-base")
+base_model = AutoModelForMaskedLM.from_pretrained("microsoft/graphcodebert-base")
+model.load_state_dict(torch.load('var_runs/model_26_2'))
+model.eval()
+base_model.eval()
+myDs=MyDataset('test.csv') 
+train_loader=DataLoader(myDs,batch_size=1,shuffle=False)
+
+
+# In[4]:
+
+
+variable_names = [
+    # One-word Variable Names
+    'count', 'value', 'result', 'flag', 'max', 'min', 'data', 'input', 'output', 'name', 'index', 'status', 'error', 'message', 'price', 'quantity', 'total', 'length', 'size', 'score',
+
+    # Two-word Variable Names
+    'studentName', 'accountBalance', 'isFound', 'maxScore', 'userAge', 'carModel', 'bookTitle', 'arrayLength', 'employeeID', 'itemPrice', 'customerAddress', 'productCategory', 'orderNumber', 'transactionType', 'bankAccount', 'shippingMethod', 'deliveryDate', 'purchaseAmount', 'inventoryItem', 'salesRevenue',
+
+    # Three-word Variable Names
+    'numberOfStudents', 'averageTemperature', 'userIsLoggedIn', 'totalSalesAmount', 'employeeSalaryRate', 'maxAllowedAttempts', 'selectedOption', 'shippingAddress', 'manufacturingDate', 'connectionPool', 'customerAccountBalance', 'employeeSalaryReport', 'productInventoryCount', 'transactionProcessingStatus', 'userAuthenticationToken', 'orderShippingAddress', 'databaseConnectionPoolSize', 'vehicleEngineTemperature', 'sensorDataProcessingRate', 'employeePayrollSystem',
+
+    # Four-word Variable Names
+    'customerAccountBalanceValue', 'employeeSalaryReportData', 'productInventoryItemCount', 'transactionProcessingStatusFlag', 'userAuthenticationTokenKey', 'orderShippingAddressDetails', 'databaseConnectionPoolMaxSize', 'vehicleEngineTemperatureReading', 'sensorDataProcessingRateLimit', 'employeePayrollSystemData', 'customerOrderShippingAddress', 'productCatalogItemNumber', 'transactionProcessingSuccessFlag', 'userAuthenticationAccessToken', 'databaseConnectionPoolConfig', 'vehicleEngineTemperatureSensor', 'sensorDataProcessingRateLimitation', 'employeePayrollSystemConfiguration', 'customerAccountBalanceHistoryData', 'transactionProcessingStatusTracking'
+]
+var_list = []
+for j in range(6):
+    d =[]
+    var_list.append(d)
+for var in variable_names:
+    try:
+        var_list[len(tokenizer.tokenize(var))-1].append(var)
+    except:
+        continue
+
+
+# In[5]:
+
+
+tot_pll = 0.0
+base_tot_pll = 0.0
+loop = tqdm(train_loader, leave=True)
+cntr = 0
+for batch in loop:
+    maxi = torch.tensor(0.0, requires_grad=True)
+    for i in range(len(batch['input_ids'])):            
+        cntr+=1
+        maski = maskis[cntr-1]
+        li = len(maski)
+        input_ids = batch['input_ids'][i][:li]
+        att_mask = batch['attention_mask'][i][:li]
+        y = n_y[cntr-1]
+        ty = tokenizer.encode(y)[1:-1]
+        num_sub_tokens_label = len(ty)
+        if num_sub_tokens_label > 6:
+            continue
+        print("Ground truth:", y)
+        m_y = random.choice(var_list[num_sub_tokens_label-1])
+        m_ty = tokenizer.encode(m_y)[1:-1]
+        print("Mock truth:", m_y)
+#            input_ids, att_mask = input_ids.to(device),att_mask.to(device)
+        outputs = model(input_ids, attention_mask = att_mask)
+        base_outputs = base_model(input_ids, attention_mask = att_mask)
+        last_hidden_state = outputs[0].squeeze()
+        base_last_hidden_state = base_outputs[0].squeeze()
+        l_o_l_sa = []
+        base_l_o_l_sa = []
+        sum_state = []
+        base_sum_state = []
+        for t in range(num_sub_tokens_label):
+            c = []
+            d = []
+            l_o_l_sa.append(c)
+            base_l_o_l_sa.append(d)
+        if len(maski) == 1:
+            masked_pos = maski[0]
+            for k in masked_pos:
+                for t in range(num_sub_tokens_label):
+                    l_o_l_sa[t].append(last_hidden_state[k+t])
+                    base_l_o_l_sa[t].append(base_last_hidden_state[k+t])
+        else:
+            for p in range(len(maski)):
+                masked_pos = maski[p]
+                for k in masked_pos:
+                    for t in range(num_sub_tokens_label):
+                        if (k+t) >= len(last_hidden_state[p]):
+                            l_o_l_sa[t].append(last_hidden_state[p+1][k+t-len(last_hidden_state[p])])
+                            base_l_o_l_sa[t].append(base_last_hidden_state[p+1][k+t-len(base_last_hidden_state[p])])
+                            continue
+                        l_o_l_sa[t].append(last_hidden_state[p][k+t])
+                        base_l_o_l_sa[t].append(base_last_hidden_state[p][k+t])
+        for t in range(num_sub_tokens_label):
+            sum_state.append(l_o_l_sa[t][0])
+            base_sum_state.append(base_l_o_l_sa[t][0])
+        for i in range(len(l_o_l_sa[0])):
+            if i == 0:
+                continue
+            for t in range(num_sub_tokens_label):
+                sum_state[t] = sum_state[t] + l_o_l_sa[t][i]
+                base_sum_state[t] = base_sum_state[t] + base_l_o_l_sa[t][i]
+        yip = len(l_o_l_sa[0])
+        val = 0.0
+        m_val = 0.0
+        m_base_val = 0.0
+        base_val = 0.0
+        for t in range(num_sub_tokens_label):
+            sum_state[t] /= yip
+            base_sum_state[t] /= yip
+            probs = F.softmax(sum_state[t], dim=0)
+            base_probs = F.softmax(base_sum_state[t], dim=0)
+            val = val - torch.log(probs[ty[t]])
+            m_val = m_val - torch.log(probs[m_ty[t]])
+            base_val = base_val - torch.log(base_probs[ty[t]])
+            m_base_val = m_base_val - torch.log(base_probs[m_ty[t]])
+        val = val / num_sub_tokens_label
+        base_val = base_val / num_sub_tokens_label
+        m_val = m_val / num_sub_tokens_label
+        m_base_val = m_base_val / num_sub_tokens_label
+        print("Sent PLL:")
+        print(val)
+        print("Base Sent PLL:")
+        print(base_val)
+        print("Net % difference:")
+        diff = (val-base_val)*100/base_val
+        print(diff)
+        tot_pll += val
+        base_tot_pll+=base_val
+        print()
+        print()
+        print("Mock Sent PLL:")
+        print(m_val)
+        print("Mock Base Sent PLL:")
+        print(m_base_val)
+        print("Mock Net % difference:")
+        m_diff = (m_val-m_base_val)*100/m_base_val
+        print(m_diff)
+        for c in sum_state:
+            del c
+        for d in base_sum_state:
+            del d
+        del sum_state
+        del base_sum_state
+        for c in l_o_l_sa:
+            del c
+        for c in base_l_o_l_sa:
+            del c
+        del l_o_l_sa
+        del base_l_o_l_sa
+        del maski
+        del input_ids
+        del att_mask
+        del last_hidden_state
+        del base_last_hidden_state
+print("Tot PLL: ", tot_pll)
+print("Base Tot PLL: ", base_tot_pll)
+
+
+# In[ ]:
+
+
+
+

stat_sampling.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[73]:
+
+
+import pandas as pd
+import numpy as np
+import torch
+from transformers import RobertaTokenizer, RobertaForSequenceClassification
+from torch import nn
+from torch.nn import init, MarginRankingLoss
+from transformers import BertModel, RobertaModel
+from transformers import BertTokenizer, RobertaTokenizer
+from torch.optim import Adam
+from distutils.version import LooseVersion
+from torch.utils.data import Dataset, DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from datetime import datetime
+from torch.autograd import Variable
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+import nltk
+import re
+import Levenshtein
+import spacy
+import en_core_web_sm
+import torch.optim as optim
+from torch.distributions import Categorical
+from numpy import linalg as LA
+from transformers import AutoModelForMaskedLM
+from nltk.corpus import wordnet
+import torch.nn.functional as F
+import random
+from transformers import get_linear_schedule_with_warmup
+from sklearn.metrics import precision_recall_fscore_support
+from nltk.corpus import words as wal
+from sklearn.utils import resample
+
+
+# In[56]:
+
+
+class MyDataset(Dataset):
+    def __init__(self,file_name):
+        df1 = pd.read_csv(file_name)
+        df1 = df1[230000:]
+        df1 = df1.fillna("")
+        res = df1['X']
+#         ab = df1['X']
+#         res = [sub.replace("<mask>", "[MASK]") for sub in ab]
+        self.X_list = res.to_numpy()
+        self.y_list = df1['y'].to_numpy()
+    def __len__(self):
+        return len(self.X_list)
+    def __getitem__(self,idx):
+        mapi = []
+        mapi.append(self.X_list[idx])
+        mapi.append(self.y_list[idx])
+        return mapi
+
+
+# In[59]:
+
+
+class Step1_model(nn.Module):
+    def __init__(self, hidden_size=512):
+        super(Step1_model, self).__init__()
+        self.hidden_size = hidden_size
+        self.tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+
+    def forward(self, mapi):
+        y = mapi[1]
+        print(y)
+        nl = re.findall(r'[A-Z](?:[a-z]+|[A-Z]*(?=[A-Z]|$))|[a-z]+|\d+', y)
+        lb = ' '.join(nl).lower()
+        x = tokenizer.tokenize(lb)
+        nlab = len(x)
+        print(nlab)
+        rand_no = random.random()
+        tok_map = {2: 0.4363429005892416,
+                   1: 0.6672580202327398,
+                   4: 0.7476060740459144,
+                   3: 0.9618703668504087,
+                   6: 0.9701028532809564,
+                   7: 0.9729244545819342,
+                   8: 0.9739508754144756,
+                   5: 0.9994508859743607,
+                   9: 0.9997507867114407,
+                   10: 0.9999112969650892,
+                   11: 0.9999788802297832,
+                   0: 0.9999831041838266,
+                   12: 0.9999873281378701,
+                   22: 0.9999957760459568,
+                   14: 1.0000000000000002}
+        for key in tok_map.keys():
+            if rand_no < tok_map[key]:
+                pred = key
+                break
+        predicted = torch.tensor([pred], dtype = float) 
+        if pred == nlab:
+            l2 = 0
+        else:
+            l2 = 1
+        actual = torch.tensor([nlab], dtype = float) 
+        l1 = Variable(torch.tensor([(actual-predicted)**2],dtype=float),requires_grad = True)
+        return {'loss':l1, 'actual_pred':pred, 'acc': l2}
+
+
+# In[60]:
+
+
+epoch_number = 0
+EPOCHS = 5
+run_int = 0
+tokenizer = AutoTokenizer.from_pretrained("microsoft/graphcodebert-base")
+model = Step1_model()
+myDs=MyDataset('dat_test.csv')
+train_loader=DataLoader(myDs,batch_size=2,shuffle=True)
+best_loss = torch.full((1,), fill_value=100000)
+
+
+flag = 0
+def train_one_epoch(transformer_model, dataset):
+    global flag
+    tot_loss1 = 0.0
+    tot_loss2 = 0.0
+    cnt = 0
+    for batch in dataset:
+        p = 0
+        inputs = batch
+        for i in range(len(inputs[0])):
+            cnt += 1
+            l = []
+            l.append(inputs[0][i])
+            l.append(inputs[1][i])
+            opi = transformer_model(l)
+            loss1 = opi['loss']
+            loss2 = opi['acc']
+            tot_loss1 += loss1
+            tot_loss2 += loss2
+
+    tot_loss1/=cnt
+    tot_loss2/=cnt
+    print('MSE  loss: ')
+    print(tot_loss1)
+    print('accuracy: ')
+    print(tot_loss2)
+    return {'MSE loss': tot_loss1, 'accuracy': tot_loss2}
+
+model.eval()
+avg_loss = train_one_epoch(model,train_loader)
+
+
+
+
+
+