Update dependencies and refactor summarization functions to include title and citation

app.py

@@ -2,8 +2,8 @@ import gradio as gr
 from main import main
 import json
 
-def rexplore_summarizer(url, id, access_key):
-    response = json.loads(main(url, id, access_key))
+def rexplore_summarizer(url, title, id, citation, access_key):
+    response = json.loads(main(url, title, id, citation, access_key))
     data = json.dumps(response, indent=4, ensure_ascii=False)
     if response["mindmap_status"] != "success":
         mindmap = "error"
@@ -15,7 +15,7 @@ def rexplore_summarizer(url, id, access_key):
         summary = response["summary"]
     return data, summary, mindmap
 
-def clear_everything(url, id, access_key, raw_data, summary, mindmap):
+def clear_everything(url, title, id, citation, access_key, raw_data, summary, mindmap):
     return None, None, None, None, None, None
 
 theme = gr.themes.Soft(
@@ -41,7 +41,9 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
     with gr.Row():
         with gr.Column():
             url = gr.Textbox(label="PDF URL", placeholder="Paste the PDF URL here")
-            id = gr.Textbox(label="DOI/arXiv ID", placeholder="Enter the DOI or arXiv ID of the document")
+            title = gr.Textbox(label="Title", placeholder="Enter the title Research Paper")
+            id = gr.Textbox(label="DOI/arXiv ID", placeholder="Enter the DOI or arXiv ID of the Research Paper")
+            citation = gr.Textbox(label="Citation", placeholder="Enter the citation of the Research Paper")
             access_key = gr.Textbox(label="Access Key", placeholder="Enter the Access Key", type="password")
             with gr.Row():
                 clear_btn = gr.Button(value="Clear", variant="stop")
@@ -53,7 +55,7 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
 
     summarize_btn.click(
         rexplore_summarizer,
-        inputs=[url, id, access_key],
+        inputs=[url, title, id, citation, access_key],
         outputs=[raw_data, summary, mindmap],
         concurrency_limit=25,
         scroll_to_output=True,
@@ -61,6 +63,6 @@ with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as ap
         api_name="rexplore_summarizer",
         show_progress="full",
     )
-    clear_btn.click(clear_everything, inputs=[url, id, raw_data, summary, mindmap, access_key], outputs=[url, id, raw_data, summary, mindmap, access_key], show_api=False)
+    clear_btn.click(clear_everything, inputs=[url, title, id, citation, raw_data, summary, mindmap, access_key], outputs=[url, id, raw_data, summary, mindmap, access_key], show_api=False)
 
 app.queue(default_concurrency_limit=25).launch(show_api=True, max_threads=500, ssr_mode=False)

main.py

@@ -1,8 +1,7 @@
 from extract_text import extract_text_from_pdf
 from math_summarizer import generate_math_summary
-from nlp_summarizer import generate_nlp_summary_and_mindmap
+from nlp_processes import generate_nlp_summary_and_mindmap
 import json
-import openai
 import dotenv
 import time
 import os
@@ -10,7 +9,7 @@ import os
 dotenv.load_dotenv()
 ACCESS_KEY = os.getenv("ACCESS_KEY")
 
-def generate_summary_mindmap(corpus):
+def generate_summary_mindmap(corpus, title, citation):
     response = {}
     math_summary = generate_math_summary(corpus)
     # print(f'As a text script expert, please help me to write a short text script with the topic \" {math_summary}\".You have three tasks, which are:\\n    1.to summarize the text I provided into a Summary .Please answer within 150-300 characters.\\n    2.to summarize the text I provided, using up to seven Highlight.\\n    3.to summarize the text I provided, using up to seven Key Insights. Each insight should include a brief in-depth analysis. Key Insight should not include timestamps.\\n    Your output should use the following template strictly, provide the results for the three tasks:\\n    ## Summary\\n    ## Highlights\\n    - Highlights\\n    ## Key Insights\\n    - Key Insights .\\n  Importantly your output must use language \"English\"')
@@ -23,25 +22,26 @@ def generate_summary_mindmap(corpus):
         response["mindmap"] = None
         return response
     else:
-        response = generate_nlp_summary_and_mindmap(math_summary)
-        print(len(response))
+        response = generate_nlp_summary_and_mindmap(math_summary, title, citation)
         return response
 
-def main(url, id, access_key):
+def main(url, title, id, citation, access_key):
     if access_key != ACCESS_KEY:
         return {"error": "Invalid Access Key", "summary": None, "mindmap": None}
     else:
         corpus = extract_text_from_pdf(url, id)
         start_time = time.time()
-        response = generate_summary_mindmap(corpus)
+        response = generate_summary_mindmap(corpus, title, citation)
         print(f"Total timetaken: {time.time() - start_time} seconds")
         return json.dumps(response, indent=4, ensure_ascii=False)
     
 if __name__ == "__main__":
     url = "https://arxiv.org/pdf/2412.21024"
     id = "123"
+    title = "Trading linearity for ellipticity: a nonsmooth approach to Einstein’s theory of gravity and the Lorentzian splitting theorems"
     access_key = "1234"
-    data = main(url, id, access_key)
+    data = main(url, title, id, access_key)
     print(len(data))
+    print(data['title'])
     with open("output.json", "w", encoding='utf-8') as f:
         json.dump(data, f, ensure_ascii=False, indent=4)

math_summarizer.py

@@ -32,7 +32,7 @@ def sanitize_text(input_string):
 def generate_math_summary(research_paper_text):
     sanitized_text = sanitize_text(research_paper_text)
     try:
-        textrank_summary = luhn_summary = lsa_summary = lexrank_summary = None
+        textrank_summary = luhn_summary = None
         def run_textrank():
             nonlocal textrank_summary
             textrank_summary = generate_textrank_summary(sanitized_text)

nlp_summarizer.py → nlp_processes.py

@@ -1,12 +1,12 @@
 from g4f.client import Client
-from g4f.Provider import RetryProvider, Blackbox, MetaAI
+from g4f.Provider import RetryProvider, Blackbox, MetaAI, BlackboxCreateAgent
 import threading
 
 def generate_nlp_summary(temp_summary):
     try:
         client = Client(
             provider=RetryProvider(
-                providers=[Blackbox, MetaAI],
+                providers=[Blackbox, MetaAI, BlackboxCreateAgent],
                 shuffle=True,
                 single_provider_retry=True,
                 max_retries=3,
@@ -28,7 +28,7 @@ def generate_nlp_mindmap(temp_summary):
     try:
         client = Client(
             provider=RetryProvider(
-                providers=[Blackbox, MetaAI],
+                providers=[Blackbox, MetaAI, BlackboxCreateAgent],
                 shuffle=True,
                 single_provider_retry=True,
                 max_retries=3,
@@ -46,8 +46,108 @@ def generate_nlp_mindmap(temp_summary):
         print(str(e))
         return False
 
-def generate_nlp_summary_and_mindmap(temp_summary):
+def fix_title(title):
+    try:
+        client = Client(
+            provider=RetryProvider(
+                providers=[Blackbox, MetaAI, BlackboxCreateAgent],
+                shuffle=True,
+                single_provider_retry=True,
+                max_retries=3,
+            ),
+        )
+        completion = client.chat.completions.create(
+            model="llama-3.1-405b",
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a highly advanced language model with strict adherence to precision and accuracy. \n\n"
+                        "Your task is to process input text, identify and correct any encoded or escaped characters, and render the text into a readable format. \n\n"
+                        "**Requirements:**\n"
+                        "1. Correctly decode and render any encoded characters (e.g., \\x sequences or LaTeX-style expressions) into their intended readable forms.\n"
+                        "2. Accurately interpret and render mathematical expressions using MathJax where appropriate.\n"
+                        "3. Produce **only the corrected sequence** as output—no additional commentary, metadata, or extraneous punctuation.\n"
+                        "4. Maintain the structure and style of the original input text, ensuring it remains true to its intended meaning and formatting.\n\n"
+                        "**Input Example:**\n"
+                        "From Painlev\\xe9 equations to ${\\cal N}=2$ susy gauge theories: prolegomena TDI-$\\\\infty$\n\n"
+                        "**Output Example:**\n"
+                        "From Painlevé equations to \\({\\cal N}=2\\) SUSY gauge theories: prolegomena TDI-\\(\\infty\\)"
+                    ),
+                },
+                {
+                    "role": "user",
+                    "content": repr(
+                        "Convert the following text into a normal, readable sequence, ensuring accurate interpretation and correction of encoded or escaped characters where necessary. "
+                        "The output must strictly adhere to the input text's original structure, maintaining readability and formatting. Use MathJax where applicable to correctly render mathematical expressions, ensuring the final sequence is flawless. "
+                        "Provide only the corrected sequence as output, with no additional commentary, formatting, or extraneous punctuation beyond what is specified in the input text.\n\n"
+                        f"**Input:** {title}\n\n"
+                    ),
+                },
+            ],
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        print(str(e))
+        return False
+
+def fix_citation(citation):
+    try:
+        client = Client(
+            provider=RetryProvider(
+                providers=[Blackbox, MetaAI, BlackboxCreateAgent],
+                shuffle=True,
+                single_provider_retry=True,
+                max_retries=3,
+            ),
+        )
+        completion = client.chat.completions.create(
+            model="llama-3.1-405b",
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "You are a highly advanced language model with strict adherence to precision and formatting. Your task is to process input text and correct any encoding errors or formatting issues, rendering it into a readable citation in APA latest edition format. \n\n"
+                        "Requirements:\n"
+                        "Accurately decode and render any encoded characters (e.g., special character codes like ’).\n"
+                        "Correctly format the citation in strict compliance with the APA latest edition guidelines.\n"
+                        "Produce only the corrected citation as output, with no additional commentary, metadata, or extraneous punctuation beyond what is specified in the text.\n"
+                        "Ensure mathematical expressions, if any, are rendered using MathJax where applicable, maintaining their proper APA context.\n"
+                        "Input Example:\n"
+                        "McCann, R. J. (2025). Trading linearity for ellipticity: a nonsmooth approach to Einstein’s theory of gravity and the Lorentzian splitting theorems (Version 1). arXiv. https://doi.org/10.48550/ARXIV.2501.00702"
+                        "Expected Output Example:\n"
+                        "McCann, R. J. (2025). Trading linearity for ellipticity: A nonsmooth approach to Einstein’s theory of gravity and the Lorentzian splitting theorems (Version 1). arXiv. https://doi.org/10.48550/ARXIV.2501.00702"
+                    ),
+                },
+                {
+                    "role": "user",
+                    "content": repr(
+                        "Convert the following text into a properly formatted citation in strict compliance with APA latest edition guidelines. Correct any encoding errors (e.g., ’) and ensure the output is clean, readable, and adheres to APA rules. Render mathematical expressions using MathJax where applicable, preserving proper context.\n"
+                        "Provide only the corrected citation as output, with no additional commentary, metadata, or extraneous punctuation beyond what is specified in the text.\n"
+                        f"**Input:** {citation}\n\n"
+                    ),
+                },
+            ],
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        print(str(e))
+        return False
+
+def generate_nlp_summary_and_mindmap(temp_summary, title, citation):
     response = {}
+    def local_fix_title():
+        fixed_title = fix_title(title)
+        if not fixed_title:
+            response["title"] = title
+        else:
+            response["title"] = fixed_title.strip()
+    def local_fix_citation():
+        fixed_citation = fix_citation(citation)
+        if not fixed_citation:
+            response["citation"] = citation
+        else:
+            response["citation"] = fixed_citation.strip
     def local_generate_nlp_summary():
         nlp_summary = generate_nlp_summary(temp_summary)
         if not nlp_summary:
@@ -67,6 +167,8 @@ def generate_nlp_summary_and_mindmap(temp_summary):
     threads = []
     threads.append(threading.Thread(target=local_generate_nlp_summary))
     threads.append(threading.Thread(target=local_generate_nlp_mindmap))
+    threads.append(threading.Thread(target=local_fix_title))
+    threads.append(threading.Thread(target=local_fix_citation))
     for thread in threads:
         thread.start()
     for thread in threads:

requirements.txt

@@ -3,4 +3,4 @@ gradio==5.8.0
 python-dotenv==1.0.1
 pdfplumber==0.11.4
 requests==2.32.3
-g4f[all]==0.4.0.2
+g4f[all]==0.4.0.4