Update app.py

app.py

@@ -1,6 +1,6 @@
 import os
 import gradio as gr
-from transformers import pipeline
+from transformers import pipeline, BertTokenizer, BertModel
 import spacy
 import lib.read_pdf
 import pandas as pd
@@ -12,6 +12,56 @@ import io
 nlp = spacy.load('en_core_web_sm')
 nlp.add_pipe('sentencizer')
 
+bert_model_name = "bert-base-uncased"
+tokenizer = BertTokenizer.from_pretrained(bert_model_name)
+model = BertModel.from_pretrained(bert_model_name)
+model.eval()
+
+import torch
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
+
+def get_bert_embeddings(texts):
+    """Obtain BERT embeddings for a list of texts."""
+    embeddings = []
+    with torch.no_grad():
+        for text in texts:
+            inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
+            outputs = model(**inputs)
+            # Take the mean of token embeddings as the sentence embedding
+            embedding = outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+            embeddings.append(embedding)
+    return np.array(embeddings)
+
+def compute_similarity(embedding1, embeddings2):
+    """Compute cosine similarity between a single embedding and a set of embeddings."""
+    return cosine_similarity([embedding1], embeddings2)[0]
+
+def compare_paragraph_to_list(paragraph, paragraph_list, top_n=3):
+    """Compare a single paragraph to a list of paragraphs and return the top N most similar ones."""
+    # Get embedding for the target paragraph
+    target_embedding = get_bert_embeddings([paragraph])[0]  # Only one paragraph
+
+    # Get embeddings for the list of paragraphs
+    list_embeddings = get_bert_embeddings(paragraph_list)
+
+    # Compute similarity between the target and each paragraph in the list
+    similarity_scores = compute_similarity(target_embedding, list_embeddings)
+
+    # Combine paragraphs with their similarity scores
+    results = [
+        {'compared_paragraph': paragraph_list[i], 'similarity_score': similarity_scores[i]}
+        for i in range(len(paragraph_list))
+    ]
+
+    # Sort the results by similarity score in descending order and take the top N
+    sorted_results = sorted(results, key=lambda x: x['similarity_score'], reverse=True)[:top_n]
+
+    # Return only the top N most similar paragraphs
+    return sorted_results
+
+
+
 def split_in_sentences(text):
     doc = nlp(text)
     return [str(sent).strip() for sent in doc.sents]
@@ -230,7 +280,8 @@ with gr.Blocks() as demo:
             with gr.Column():
                 gr.Markdown("### PDF 1 Analysis")
                 selected_paragraph_1 = gr.Textbox(label="Selected Paragraph 1 Content", lines=4)
-                selected_paragraph_1.change(show, paragraph_1_dropdown, selected_paragraph_1)
+                selected_paragraph_btn1 = gr.Button("Nearest paragraph content from pdf 2")
+                selected_paragraph_btn1.click(fn=lambda p: compare_paragraph_to_list(p, stored_paragraphs_2), inputs=paragraph_1_dropdown, outputs=selected_paragraph_1)
                 summarize_btn1 = gr.Button("Summarize Text from PDF 1")
                 summary_textbox_1 = gr.Textbox(label="Summary for PDF 1", lines=2)
                 summarize_btn1.click(fn=lambda p: process_paragraph_1_sum(p), inputs=paragraph_1_dropdown, outputs=summary_textbox_1)