Update app.py

app.py

@@ -182,27 +182,28 @@ if uploaded_file is not None:
             sentences = sentence_tokenize(text)
             for sent in sentences:
               x = mask_generation(sent)
-              sent_out = re.sub("\n","|",sent)
-              sent_out = re.sub(".","|",sent_out)
-              sent_out = re.sub(",","|",sent_out)
-              sent_out = re.sub(" ","|",sent_out)
-              sent_n_q_c=sent_out.split("|")
-              # sent_n_q_c=[]  
-              # sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
-              # for i in sent_n:
-              #     for j in i.split(" "):
-              #         sent_n_q_c+=j.split(',')
-              # x_q = x.lower().replace('.',' ').split(' ')    
-              # e=[]
-              # for i in x_q:
-              #    e+=i.split(',')           
-              sent_out = re.sub("\n","|",x)
-              sent_out = re.sub(".","|",sent_out)
-              sent_out = re.sub(",","|",sent_out)
-              sent_out = re.sub(" ","|",sent_out)   
-              e = sent_out.split("|")       
-              t5_words=set(sent_n_q_c).difference(set(e))      
+              # sent_out = re.sub("\n","|",sent)
+              # sent_out = re.sub(".","|",sent_out)
+              # sent_out = re.sub(",","|",sent_out)
+              # sent_out = re.sub(" ","|",sent_out)
+              # sent_n_q_c=sent_out.split("|")
+              sent_n_q_c=[]  
+              sent_n = list(set(sent.lower().replace('.',' ').split("\n")))
+              for i in sent_n:
+                  for j in i.split(" "):
+                      sent_n_q_c+=j.split(',')
+              x_q = x.lower().replace('.',' ').split(' ')    
+              e=[]
+              for i in x_q:
+                 e+=i.split(',')           
+              # sent_out = re.sub("\n","|",x)
+              # sent_out = re.sub(".","|",sent_out)
+              # sent_out = re.sub(",","|",sent_out)
+              # sent_out = re.sub(" ","|",sent_out)   
+              # e = sent_out.split("|")       
+              t5_words=list(set(sent_n_q_c).difference(set(e)))
               entities,words_out = extract_entities(sent)
+              words_out+=t5_words
               # print("\nwords_out:",words_out)
               # print("\nT5",t5_words)
               # print("X:",x,"\nsent:",sent,"\nx_q:",x_q,"\nsent_n:",sent_n,"\ne:",e,"\nsent_n_q_c:",sent_n_q_c,'\nt5_words',t5_words)
@@ -211,7 +212,7 @@ if uploaded_file is not None:
               new=[]
               for w in words_out:
                 new+=w.split('\n')
-              words_out+=t5_words
+              # words_out+=t5_words
               new+=bert_words
               words_out = [i for i in new if len(i)>3]
               # print("\nfinal:",words_out)