Add initial project structure with PDF summarization features and Gradio interface

.gitignore

@@ -0,0 +1 @@
+__pycache__

README.md

@@ -1,6 +1,6 @@
 ---
 title: Rexplore Api
-emoji: 🌖
+emoji: 📑
 colorFrom: red
 colorTo: yellow
 sdk: gradio

app.py

@@ -0,0 +1,54 @@
+import gradio as gr
+from main import main
+
+def rexplore_summarizer(corpus):
+    response = main(corpus)
+    return response, response['summary'], response['mindmap']
+
+def clear_everything(text_corpus, raw_data, summary, mindmap):
+    return None, None, None, None
+
+theme = gr.themes.Soft(
+    primary_hue="purple",
+    secondary_hue="cyan",
+    neutral_hue="slate",
+    font=[
+        gr.themes.GoogleFont('Syne'), 
+        gr.themes.GoogleFont('Poppins'), 
+        gr.themes.GoogleFont('Poppins'), 
+        gr.themes.GoogleFont('Poppins')
+    ],
+)
+
+with gr.Blocks(theme=theme, title="ReXplore Summarizer", fill_height=True) as app:
+    gr.HTML(
+        value ='''
+        <h1 style="text-align: center;">ReXplore Summarizer <p style="text-align: center;">Designed and Developed by <a href='https://raannakasturi.eu.org' target="_blank" rel="nofollow noreferrer external">Nayan Kasturi</a></p> </h1>
+        <p style="text-align: center;">This app uses a hybrid approach to summarize PDF documents based on CPU as well as GPU.</p>
+        <p style="text-align: center;">The app uses traditional methodologies such as TextRank, LSA, Luhn algorithms as well as large language model (LLM) to generate summaries as well as mindmaps.</p>
+        <p style="text-align: center;">The summarization process can take some time depending on the size of the text corpus and the complexity of the content.</p>
+        ''')
+    with gr.Row():
+        with gr.Column():
+            text_corpus = gr.TextArea(label="Text Corpus", placeholder="Paste the text corpus here", lines=5)
+            with gr.Row():
+                clear_btn = gr.Button(value="Clear", variant='stop')
+                summarize_btn = gr.Button(value="Summarize", variant='primary')
+        raw_data = gr.TextArea(label="Raw Data", placeholder="The generated raw data will be displayed here", lines=7, interactive=False, show_copy_button=True)
+    with gr.Row():
+        summary = gr.TextArea(label="Summary", placeholder="The generated summary will be displayed here", lines=7, interactive=False, show_copy_button=True)
+        mindmap = gr.TextArea(label="Mindmap", placeholder="The generated mindmap will be displayed here", lines=7, interactive=False, show_copy_button=True)
+
+    summarize_btn.click(
+        rexplore_summarizer,
+        inputs=[text_corpus],
+        outputs=[raw_data, summary, mindmap],
+        concurrency_limit=1,
+        scroll_to_output=True,
+        show_api=True,
+        api_name="rexplore_summarizer",
+        show_progress="full",
+    )
+    clear_btn.click(clear_everything, inputs=[text_corpus, raw_data, summary, mindmap], outputs=[text_corpus, raw_data, summary, mindmap], show_api=False)
+
+app.queue(default_concurrency_limit=1).launch(show_api=True)

main.py

@@ -0,0 +1,35 @@
+from math_summarizer import generate_math_summary
+from nlp_summarizer import generate_nlp_summary_and_mindmap
+from tools import parse_pdf
+import openai
+import time
+
+def create_client(api_key):
+    client = openai.OpenAI(
+        api_key=api_key,
+        base_url="https://glhf.chat/api/openai/v1",
+    )
+    return client
+
+def generate_summary(client, corpus):
+    response = {}
+    print("Generating Math Summary")
+    math_summary = generate_math_summary(corpus)
+    if not math_summary:
+        print("Error generating Math Summary")
+        response['summary_status'] = "error"
+        response['summary'] = None
+        response['mindmap_status'] = "success"
+        response['mindmap'] = None
+        return response
+    else:
+        print("Math Summary Generated Successfully")
+        print("Generating NLP Summary and Mindmap")
+        response = generate_nlp_summary_and_mindmap(client, corpus)
+        print("NLP Summary and Mindmap Generated Successfully")
+        return response
+
+def main(corpus):
+    client = create_client("glhf_0d3c695626fec941eeb7914dd0e36da5")
+    response = generate_summary(client, corpus)
+    return response

math_summarizer.py

@@ -0,0 +1,91 @@
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.text_rank import TextRankSummarizer
+from sumy.summarizers.luhn import LuhnSummarizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.summarizers.lsa import LsaSummarizer
+from sumy.nlp.stemmers import Stemmer
+from sumy.utils import get_stop_words
+import nltk
+import threading
+
+LANGUAGE = "english"
+SENTENCES_COUNT = 15
+
+def generate_textrank_summary(research_paper_text):
+    nltk.download('punkt', quiet=True)
+    nltk.download('punkt_tab', quiet=True)
+    parser = PlaintextParser.from_string(research_paper_text, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = TextRankSummarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    sentences = summarizer(parser.document, SENTENCES_COUNT)
+    summary = ""
+    for sentence in sentences:
+        summary += str(sentence) + ""
+    return summary
+
+def generate_luhn_summary(research_paper_text):
+    nltk.download('punkt', quiet=True)
+    nltk.download('punkt_tab', quiet=True)
+    parser = PlaintextParser.from_string(research_paper_text, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = LuhnSummarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    sentences = summarizer(parser.document, SENTENCES_COUNT)
+    summary = ""
+    for sentence in sentences:
+        summary += str(sentence) + ""
+    return summary
+
+def generate_lsa_summary(research_paper_text):
+    nltk.download('punkt', quiet=True)
+    nltk.download('punkt_tab', quiet=True)
+    parser = PlaintextParser.from_string(research_paper_text, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = LsaSummarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    sentences = summarizer(parser.document, SENTENCES_COUNT)
+    summary = ""
+    for sentence in sentences:
+        summary += str(sentence) + ""
+    return summary
+
+def sanitize_text(input_string):
+    try:
+        encoded_bytes = input_string.encode('utf-8')
+        decoded_string = encoded_bytes.decode('utf-8')
+        return decoded_string
+    except UnicodeEncodeError as e:
+        print(f"Encoding error: {e}")
+        raise
+    except UnicodeDecodeError as e:
+        print(f"Decoding error: {e}")
+        raise
+
+def generate_math_summary(research_paper_text):
+    sanitized_text = sanitize_text(research_paper_text)
+    try:
+        textrank_summary = luhn_summary = lsa_summary = None
+        def run_textrank():
+            nonlocal textrank_summary
+            textrank_summary = generate_textrank_summary(sanitized_text)
+        def run_luhn():
+            nonlocal luhn_summary
+            luhn_summary = generate_luhn_summary(sanitized_text)
+        def run_lsa():
+            nonlocal lsa_summary
+            lsa_summary = generate_lsa_summary(sanitized_text)
+        threads = []
+        threads.append(threading.Thread(target=run_textrank))
+        threads.append(threading.Thread(target=run_luhn))
+        threads.append(threading.Thread(target=run_lsa))
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        math_summary = textrank_summary.replace("\n", "") + f"\n {'-'*30} \n" + luhn_summary.replace("\n", "") + f"\n {'-'*30} \n" + lsa_summary.replace("\n", "")
+        return math_summary
+    except Exception as e:
+        print(e)
+        return False

nlp_summarizer.py

@@ -0,0 +1,56 @@
+import threading
+
+def generate_nlp_summary(client, temp_summary):
+    print("Generating NLP Summary")
+    try:
+        completion = client.chat.completions.create(
+            model="hf:meta-llama/Meta-Llama-3.1-405B-Instruct",
+            messages=[
+                {"role": "system", "content": "You are a helpful research assistant for generating well-formatted summaries from scientific research papers."},
+                {"role": "user", "content": f'As a text script expert, please help me to write a short text script with the topic \" {temp_summary}\".You have three tasks, which are:\\n    1.to summarize the text I provided into a Summary .Please answer within 150-300 characters.\\n    2.to summarize the text I provided, using up to seven Highlight.\\n    3.to summarize the text I provided, using up to seven Key Insights. Each insight should include a brief in-depth analysis. Key Insight should not include timestamps.\\n    Your output should use the following template strictly, provide the results for the three tasks:\\n    ## Summary\\n    ## Highlights\\n    - Highlights\\n    ## Key Insights\\n    - Key Insights .\\n  Importantly your output must use language \"English\"'}
+            ]
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        return False
+
+def generate_nlp_mindmap(client, temp_summary):
+    print("Generating NLP Mindmap")
+    try:
+        completion = client.chat.completions.create(
+            model="hf:meta-llama/Meta-Llama-3.1-405B-Instruct",
+            messages=[
+                {"role": "system", "content": "You are a helpful research assistant for generating well-formatted mindmaps from scientific research papers."},
+                {"role": "user", "content": f'As a text script expert, please help me to write a short text script with the topic \"{temp_summary}\".Your output should use the following template:\\n\\n## {{Subtitle01}}\\n- {{Bulletpoint01}}\\n- {{Bulletpoint02}}\\n## {{Subtitle02}}\\n- {{Bulletpoint03}}\\n- {{Bulletpoint04}}\\n\\nSummarize the giving topic to generate a mind map (as many subtitles as possible, with a minimum of three subtitles) structure markdown. Do not include anything in the response, that is not the part of mindmap.\\n  Most Importantly your output must use language \"English\" and each point or pointer should include no more than 9 words.'}
+            ]
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        return False
+
+def generate_nlp_summary_and_mindmap(client, temp_summary):
+    response = {}
+    def local_generate_nlp_summary():
+        nlp_summary = generate_nlp_summary(client, temp_summary)
+        if not nlp_summary:
+            response['summary_status'] = "error"
+            response['summary'] = None
+        else:
+            response['summary_status'] = "success"
+            response['summary'] = nlp_summary
+    def local_generate_nlp_mindmap():
+        nlp_mindmap = generate_nlp_mindmap(client, temp_summary)
+        if not nlp_mindmap:
+            response['mindmap_status'] = "error"
+            response['mindmap'] = None
+        else:
+            response['mindmap_status'] = "success"
+            response['mindmap'] = nlp_mindmap
+    threads = []
+    threads.append(threading.Thread(target=local_generate_nlp_summary))
+    threads.append(threading.Thread(target=local_generate_nlp_mindmap))
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+    return response

requirements.txt

@@ -0,0 +1 @@
+openai

tools.py

@@ -0,0 +1,19 @@
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def parse_pdf(pdf_path):
+    loader = PyPDFLoader(pdf_path)
+    pages = loader.load_and_split()
+    all_text = " ".join([page.page_content for page in pages])
+    start_index = all_text.find("ABSTRACT")
+    end_index = all_text.find("REFERENCES")
+    if start_index != -1 and end_index != -1 and start_index < end_index:
+        relevant_text = all_text[start_index:end_index]
+    else:
+        relevant_text = all_text
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=250, chunk_overlap=50)
+    text_list = text_splitter.split_text(relevant_text)
+    research_paper_text = "".join(text_list)
+    length_of_research_paper = len(research_paper_text)
+    print(f"Length of Research Paper: {length_of_research_paper}")
+    return research_paper_text