Update app.py

app.py

@@ -1,4 +1,9 @@
-# import parsing # decomment to download data from the website and parse it
+# import parsing # decomment to download data from the website and parse it # string, gensim, tqdm, transformers, torch
+from string import punctuation
+from tqdm.auto import tqdm, trange
+
+import torch
+from transformers import AutoTokenizer, AutoModel
 
 import pandas as pd
 import nltk
@@ -12,6 +17,8 @@ from nltk import pos_tag  # for parts of speech
 from sklearn.metrics import pairwise_distances  # cosine similarity
 from nltk import word_tokenize
 from nltk.corpus import stopwords
+from gensim.models import Word2Vec, KeyedVectors
+import gensim.downloader as api
 from sklearn.metrics.pairwise import cosine_similarity
 import gradio as gr
 import time
@@ -74,6 +81,7 @@ x_tfidf = tfidf.fit_transform(df['Normalized and StopWords question']).toarray()
 features_tfidf = tfidf.get_feature_names_out()  # use function to get all the normalized words
 df_tfidf = pd.DataFrame(x_tfidf, columns = features_tfidf)  # create dataframe to show the 0, 1 value for each word
 
+# bot tf idf algorithm without context
 def chat_tfidf(question):
     tidy_question = text_normalization(removeStopWords(question))  # clean & lemmatize the question
     tf = tfidf.transform([tidy_question]).toarray()  # convert the question into a vector
@@ -83,23 +91,239 @@ def chat_tfidf(question):
     answer = df['answer'].loc[index_value]
     return answer
 
+# bot tf idf algorithm with context
+def chat_tfidf_context(question, history):
+  
+  len_history = len(history)
+
+  if len_history > 1:
+    memory_weights = np.array([0.1, 0.3, 1.0]) # .reshape((3,1))
+    # take last two sentences in accordance to bot's memory
+    history = history[-2:]
+
+  else:
+    memory_weights = np.array([0.3, 1.0])
+
+  history_sentence = np.zeros(shape=(len_history+1, 5024))
+
+  for ind, h in enumerate(history):
+    # normalize first question from context      
+    tidy_question = text_normalization(removeStopWords(h[0]))
+    # pass via tfidf
+    tf = tfidf.transform([tidy_question]).toarray()
+
+    # assign tf idf vector to history sentence
+    history_sentence[ind] = tf * memory_weights[ind]
+  
+  tidy_question = text_normalization(removeStopWords(question))
+  tf = tfidf.transform([tidy_question]).toarray()
+
+  history_sentence[-1] = tf
+  history_sentence = history_sentence.mean(axis=0).reshape(1,-1)
+
+  cos = 1- pairwise_distances(df_tfidf, history_sentence, metric = 'cosine')
+  index_value = cos.argmax()
+  answer = df['answer'].loc[index_value]
+
+  return answer  
+#------------------------------------------------------------------------------------------------#
+punkt = [p for p in punctuation] + ["`", "``" ,"''", "'"]
+
+def tokenize(sent: str) -> str:
+  tokens = nltk.word_tokenize(sent.lower())  # tokenize words
+  return ' '.join([word for word in tokens if word not in stop and word not in punkt])
+
+questions_preprocessed = []
+for question in df["question"].tolist() + df["answer"].tolist():
+  questions_preprocessed.append(tokenize(question))
+    
+questions_w2v = [sent.split(" ") for sent in questions_preprocessed]
+
+w2v = KeyedVectors.load('w2v.bin')
+unknown_vector = np.random.uniform(low=-0.2, high=0.2, size=(25,))
+
+# define function to form sentences with w2v
+def w2v_get_vector_for_sentence(sentence):
+  sent = nltk.word_tokenize(sentence.lower())
+  sent = [word for word in sent if word not in punkt]
+  sentence_vector = []
+  if len(sent)==0:
+    sentence_vector.append(unknown_vector)
+  else:
+    for word in sent:
+      if word in w2v.key_to_index:
+        sentence_vector.append(w2v[word])
+      else:
+        sentence_vector.append(unknown_vector)
+
+  return np.array(sentence_vector).mean(axis=0)
+
+# create base for w2v
+base = np.zeros(shape=(len(df.question), 25))
+for ind, sentence in enumerate(df['question']):  #  df[df['question'].str.len() >= 1]
+  base[ind] = w2v_get_vector_for_sentence(sentence)
+
+# bot w2v algorithm without context
+def chat_word2vec(question):
+  question = [w2v_get_vector_for_sentence(question)]
+  cos = 1-pairwise_distances(base, question, metric = 'cosine')  # calculate the cosine value
+  index_value = cos.argmax()  # find the index of the maximum cosine value
+  answer = df['answer'].loc[index_value]
+  return answer
+    
+# bot w2v algorithm with context
+def chat_word2vec_context(question, history):
+  
+  len_history = len(history)
+
+  if len_history > 1:
+    memory_weights = np.array([0.1, 0.3, 1.0]) # .reshape((3,1))
+    # take last two sentences in accordance to bot's memory
+    history = history[-2:]
+
+  else:
+    memory_weights = np.array([0.3, 1.0])
+
+  history_sentence = np.zeros(shape=(len_history+1, 25))
+
+  for ind, h in enumerate(history):
+    sentence = w2v_get_vector_for_sentence(h[0])
+    history_sentence[ind] = sentence * memory_weights[ind]
+  
+  question = w2v_get_vector_for_sentence(question)
+
+  history_sentence[-1] = question
+  history_sentence = history_sentence.mean(axis=0).reshape(1, -1)
+
+  cos = 1-pairwise_distances(base, history_sentence, metric = 'cosine')
+  index_value = cos.argmax()
+  answer = df['answer'].loc[index_value]
 
+  return answer  
 
+#------------------------------------------------------------------------------------------------#
+# Let's try bert model by elastic and with e5
+model_name = "elastic/multilingual-e5-small-optimized"
+device = "cpu"
 
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModel.from_pretrained(model_name)
 
+class BERTSearchEngine:
+    def __init__(self, model, tokenizer):
+        self.raw_procesed_data = [self.preprocess(sample, tokenizer) for sample in text_database]
+        self.base = []
+        self.retriever = None
+        self.inverted_index = {}
+        self._init_retriever(model, tokenizer, text_database)
+        self._init_inverted_index(text_database)
+
+    @staticmethod
+    def preprocess(sentence: str, tokenizer):
+        return tokenizer(sentence, padding=True, truncation=True, return_tensors='pt')
+
+    def _embed_bert_cls(self, tokenized_text: dict[torch.Tensor]) -> np.array:
+        with torch.no_grad():
+            model_output = self.retriever(**{k: v.to(self.retriever.device) for k, v in tokenized_text.items()})
+        embeddings = model_output.last_hidden_state[:, 0, :]
+        embeddings = torch.nn.functional.normalize(embeddings)
+        return embeddings[0].cpu().numpy()
+
+    def _init_retriever(self, model, tokenizer, text_database):
+        self.retriever = model
+        self.tokenizer = tokenizer
+        self.base = None #np.array([self._embed_bert_cls(self.preprocess(text, tokenizer)) for text in tqdm(text_database)])
+
+    def retrieve(self, query: str) -> np.array:
+        return self._embed_bert_cls(self.preprocess(query, self.tokenizer))
+
+    def retrieve_documents(self, query: str, top_k=3) -> list[int]:
+        query_vector = self.retrieve(query)
+        cosine_similarities = cosine_similarity([query_vector], self.base).flatten()
+        relevant_indices = np.argsort(cosine_similarities, axis=0)[::-1][:top_k]
+        return relevant_indices.tolist()
+
+    def _init_inverted_index(self, text_database: list[str]):
+        self.inverted_index = dict(enumerate(text_database))
+
+    def display_relevant_docs(self, query, full_database, top_k=3) -> list[int]:
+        docs_indexes = self.retrieve_documents(query, top_k=top_k)
+        return [self.inverted_index[ind] for ind in docs_indexes]
+
+    def find_answer(self, query: str) -> int:
+        query_vector = self.retrieve(query)
+        cosine_similarities = cosine_similarity([query_vector], self.base).flatten()
+        relevant_indice = np.argmax(cosine_similarities, axis=0)
+        return relevant_indice
+
+simple_search_engine = BERTSearchEngine(model, tokenizer)
+simple_search_engine.bert = np.load(bert_base.npy)
+
+# bot bert algorithm without context
+def chat_bert(question):
+  ind = simple_search_engine.find_answer(question)
+  answer = df['answer'].iloc[ind]
+  return answer
+
+# bot bert algorithm with context
+def chat_bert_context(question, history):
+  
+  len_history = len(history)
+
+  if len_history > 1:
+    memory_weights = np.array([0.1, 0.3, 1.0]) # .reshape((3,1))
+    # take last two sentences in accordance to bot's memory
+    history = history[-2:]
+
+  else:
+    memory_weights = np.array([0.3, 1.0])
+
+  history_sentence = np.zeros(shape=(len_history+1, 384))
+
+  for ind, h in enumerate(history):
+
+    sentence = simple_search_engine.retrieve(h)
+    history_sentence[ind] = sentence * memory_weights[ind]
+  
+  question = simple_search_engine.retrieve(question)
+
+  history_sentence[-1] = question
+  history_sentence = history_sentence.mean(axis=0).reshape(1, -1)
+
+  cosine_similarities = cosine_similarity(history_sentence, simple_search_engine.base).flatten()
+  relevant_indice = np.argmax(cosine_similarities, axis=0)
+  answer = df['answer'].loc[relevant_indice]
+
+  return answer  
+#------------------------------------------------------------------------------------------------#
+# gradio part 
 def echo(message, history, model):
-  print(model)
-  print(history)
+  # print(model)
+  # print(history)
+  # if model=="TF-IDF":
+  #   answer = chat_tfidf(message)
+  #   return answer
+
+  # elif model=="W2V":
+  #   answer = chat_word2vec(message)
+  #   return answer
+
+  # elif model=="BERT":
+  #   answer = chat_bert(message)
+  #   return answer
+    
   if model=="TF-IDF":
-    answer = chat_tfidf(message)
+    # answer = chat_tfidf(message)
+    answer = chat_tfidf_context(message, history)
     return answer
 
   elif model=="W2V":
-    answer = chat_word2vec(message)
+    # answer = chat_word2vec(message)
+    answer = chat_word2vec_context(message, history)
     return answer
 
   elif model=="BERT":
-    answer = chat_bert(message)
+    answer = chat_bert_context(message, history)
     return answer