Refactor summarizer functions to accept URL, ID, and access key; implement PDF text extraction

app.py

@@ -1,12 +1,12 @@
 import gradio as gr
 from main import main
 
-def rexplore_summarizer(corpus):
-    response = main(corpus)
+def rexplore_summarizer(url, id, access_key):
+    response = main(url, id, access_key)
     return response, response['summary'], response['mindmap']
 
-def clear_everything(text_corpus, raw_data, summary, mindmap):
-    return None, None, None, None
+def clear_everything(url, id, access_key, raw_data, summary, mindmap):
+    return None, None, None, None, None, None
 
 theme = gr.themes.Soft(
     primary_hue="purple",
@@ -30,7 +30,9 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
         ''')
     with gr.Row():
         with gr.Column():
-            text_corpus = gr.TextArea(label="Text Corpus", placeholder="Paste the text corpus here", lines=5)
+            url = gr.Textbox(label="PDF URL", placeholder="Paste the PDF URL here")
+            id = gr.Textbox(label="DOI/arXiv ID", placeholder="Enter the DOI or arXiv ID of the document")
+            access_key = gr.Textbox(label="Access Key", placeholder="Enter the Access Key")
             with gr.Row():
                 clear_btn = gr.Button(value="Clear", variant='stop')
                 summarize_btn = gr.Button(value="Summarize", variant='primary')
@@ -41,7 +43,7 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
 
     summarize_btn.click(
         rexplore_summarizer,
-        inputs=[text_corpus],
+        inputs=[url, id, access_key],
         outputs=[raw_data, summary, mindmap],
         concurrency_limit=25,
         scroll_to_output=True,
@@ -49,6 +51,6 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
         api_name="rexplore_summarizer",
         show_progress="full",
     )
-    clear_btn.click(clear_everything, inputs=[text_corpus, raw_data, summary, mindmap], outputs=[text_corpus, raw_data, summary, mindmap], show_api=False)
+    clear_btn.click(clear_everything, inputs=[url, id, raw_data, summary, mindmap, access_key], outputs=[url, id, raw_data, summary, mindmap, access_key], show_api=False)
 
 app.queue(default_concurrency_limit=25).launch(show_api=True)

extract_text.py

@@ -0,0 +1,34 @@
+from pdfplumber import open as pdf_open
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import requests
+import os
+
+def download_pdf(url, id):
+    file_path = f"{id}.pdf"
+    response = requests.get(url)
+    with open(file_path, 'wb') as file:
+        file.write(response.content)
+    return file_path
+
+def extract_text_from_pdf(url, id):
+    pdf_path = download_pdf(url, id)
+    try:
+        with pdf_open(pdf_path) as pdf:
+            all_text = ""
+            for page in pdf.pages:
+                all_text += page.extract_text() + " "
+        start_index = all_text.find("ABSTRACT")
+        end_index = all_text.find("REFERENCES")
+        if start_index != -1 and end_index != -1 and start_index < end_index:
+            relevant_text = all_text[start_index:end_index]
+        else:
+            relevant_text = all_text
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=250, chunk_overlap=50)
+        text_list = text_splitter.split_text(relevant_text)
+        research_paper_text = "".join(text_list)
+    except Exception as e:
+        print(f"Error processing PDF: {e}")
+        research_paper_text = ""
+    finally:
+        os.remove(pdf_path)
+    return research_paper_text

main.py

@@ -1,3 +1,4 @@
+from extract_text import extract_text_from_pdf
 from math_summarizer import generate_math_summary
 from nlp_summarizer import generate_nlp_summary_and_mindmap
 import openai
@@ -7,6 +8,7 @@ import os
 
 dotenv.load_dotenv()
 API_KEY = os.getenv('API_KEY')
+ACCESS_KEY = os.getenv('ACCESS_KEY')
 
 def create_client(api_key):
     client = openai.OpenAI(
@@ -29,9 +31,13 @@ def generate_summary(client, corpus):
         response = generate_nlp_summary_and_mindmap(client, corpus)
         return response
 
-def main(corpus):
-    start_time = time.time()
-    client = create_client(API_KEY)
-    response = generate_summary(client, corpus)
-    print(f"Total timetaken: {time.time() - start_time} seconds")
-    return response
+def main(url, id, access_key):
+    if access_key != ACCESS_KEY:
+        return {"error": "Invalid Access Key", "summary": None, "mindmap": None}
+    else:
+        corpus = extract_text_from_pdf(url, id)
+        start_time = time.time()
+        client = create_client(API_KEY)
+        response = generate_summary(client, corpus)
+        print(f"Total timetaken: {time.time() - start_time} seconds")
+        return response

requirements.txt

@@ -1,3 +1,6 @@
 openai==1.57.3
 gradio==5.8.0
 python-dotenv==1.0.1
+pdfplumber==0.11.4
+langchain==0.3.13
+requests==2.32.3