Update .gitignore, refactor NLP summary functions, and enhance output handling

.gitignore

@@ -1,3 +1,7 @@
 __pycache__
 .env
-/downloads
+/downloads
+*.html
+/har*
+*.json
+test*

main.py

@@ -8,19 +8,13 @@ import time
 import os
 
 dotenv.load_dotenv()
-API_KEY = os.getenv("API_KEY")
 ACCESS_KEY = os.getenv("ACCESS_KEY")
 
-def create_client(api_key):
-    client = openai.OpenAI(
-        api_key=api_key,
-        base_url="https://api.groq.com/openai/v1",
-    )
-    return client
-
-def generate_summary(client, corpus):
+def generate_summary_mindmap(corpus):
     response = {}
     math_summary = generate_math_summary(corpus)
+    # print(f'As a text script expert, please help me to write a short text script with the topic \" {math_summary}\".You have three tasks, which are:\\n    1.to summarize the text I provided into a Summary .Please answer within 150-300 characters.\\n    2.to summarize the text I provided, using up to seven Highlight.\\n    3.to summarize the text I provided, using up to seven Key Insights. Each insight should include a brief in-depth analysis. Key Insight should not include timestamps.\\n    Your output should use the following template strictly, provide the results for the three tasks:\\n    ## Summary\\n    ## Highlights\\n    - Highlights\\n    ## Key Insights\\n    - Key Insights .\\n  Importantly your output must use language \"English\"')
+    # exit()
     if not math_summary:
         print("Error generating Math Summary")
         response["summary_status"] = "error"
@@ -29,7 +23,8 @@ def generate_summary(client, corpus):
         response["mindmap"] = None
         return response
     else:
-        response = generate_nlp_summary_and_mindmap(client, corpus)
+        response = generate_nlp_summary_and_mindmap(math_summary)
+        print(len(response))
         return response
 
 def main(url, id, access_key):
@@ -38,13 +33,15 @@ def main(url, id, access_key):
     else:
         corpus = extract_text_from_pdf(url, id)
         start_time = time.time()
-        client = create_client(API_KEY)
-        response = generate_summary(client, corpus)
+        response = generate_summary_mindmap(corpus)
         print(f"Total timetaken: {time.time() - start_time} seconds")
         return json.dumps(response, indent=4, ensure_ascii=False)
     
 if __name__ == "__main__":
-    url = "https://arxiv.org/pdf/2106.01484"
+    url = "https://arxiv.org/pdf/2412.21024"
     id = "123"
     access_key = "1234"
-    print(main(url, id, access_key))
+    data = main(url, id, access_key)
+    print(len(data))
+    with open("output.json", "w", encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)

nlp_summarizer.py

@@ -1,37 +1,57 @@
+from g4f.client import Client
+from g4f.Provider import RetryProvider, Blackbox, MetaAI
 import threading
 
-def generate_nlp_summary(client, temp_summary):
-    print("Generating NLP Summary")
+def generate_nlp_summary(temp_summary):
     try:
+        client = Client(
+            provider=RetryProvider(
+                providers=[Blackbox, MetaAI],
+                shuffle=True,
+                single_provider_retry=True,
+                max_retries=3,
+            ),
+        )
         completion = client.chat.completions.create(
-            model="llama-3.2-90b-vision-preview",
+            model="llama-3.1-405b",
             messages=[
                 {"role": "system", "content": "You are a helpful research assistant for generating well-formatted summaries from scientific research papers."},
                 {"role": "user", "content": f'As a text script expert, please help me to write a short text script with the topic \" {temp_summary}\".You have three tasks, which are:\\n    1.to summarize the text I provided into a Summary .Please answer within 150-300 characters.\\n    2.to summarize the text I provided, using up to seven Highlight.\\n    3.to summarize the text I provided, using up to seven Key Insights. Each insight should include a brief in-depth analysis. Key Insight should not include timestamps.\\n    Your output should use the following template strictly, provide the results for the three tasks:\\n    ## Summary\\n    ## Highlights\\n    - Highlights\\n    ## Key Insights\\n    - Key Insights .\\n  Importantly your output must use language \"English\"'}
-            ]
+            ],
         )
-        return completion.choices[0].message.content.replace("**", "")
+        print(completion.choices[0].message.content.replace("**", "").replace("\n\n", "\n"))
+        return completion.choices[0].message.content.replace("**", "").replace("\n\n", "\n")
     except Exception as e:
+        print(str(e))
         return False
 
-def generate_nlp_mindmap(client, temp_summary):
-    print("Generating Mindmap")
+def generate_nlp_mindmap(temp_summary):
     try:
+        client = Client(
+            provider=RetryProvider(
+                providers=[Blackbox, MetaAI],
+                shuffle=True,
+                single_provider_retry=True,
+                max_retries=3,
+            ),
+        )
         completion = client.chat.completions.create(
-            model="llama-3.2-90b-vision-preview",
+            model="llama-3.1-405b",
             messages=[
                 {"role": "system", "content": "You are a helpful research assistant for generating well-formatted mindmaps from scientific research papers."},
                 {"role": "user", "content": f'As a text script expert, please help me to write a short text script with the topic \"{temp_summary}\".Your output should use the following template:\\n\\n## {{Subtitle01}}\\n- {{Bulletpoint01}}\\n- {{Bulletpoint02}}\\n## {{Subtitle02}}\\n- {{Bulletpoint03}}\\n- {{Bulletpoint04}}\\n\\nSummarize the giving topic to generate a mind map (as many subtitles as possible, with a minimum of three subtitles) structure markdown. Do not include anything in the response, that is not the part of mindmap.\\n  Most Importantly your output must use language \"English\" and each point or pointer should include no more than 9 words.'}
-            ]
+            ],
         )
-        return completion.choices[0].message.content.replace("**", "")
+        print(completion.choices[0].message.content.replace("**", "").replace("\n\n", "\n"))
+        return completion.choices[0].message.content.replace("**", "").replace("\n\n", "\n")
     except Exception as e:
+        print(str(e))
         return False
 
-def generate_nlp_summary_and_mindmap(client, temp_summary):
+def generate_nlp_summary_and_mindmap(temp_summary):
     response = {}
     def local_generate_nlp_summary():
-        nlp_summary = generate_nlp_summary(client, temp_summary)
+        nlp_summary = generate_nlp_summary(temp_summary)
         if not nlp_summary:
             response["summary_status"] = "error"
             response["summary"] = None
@@ -39,7 +59,7 @@ def generate_nlp_summary_and_mindmap(client, temp_summary):
             response["summary_status"] = "success"
             response["summary"] = nlp_summary
     def local_generate_nlp_mindmap():
-        nlp_mindmap = generate_nlp_mindmap(client, temp_summary)
+        nlp_mindmap = generate_nlp_mindmap(temp_summary)
         if not nlp_mindmap:
             response["mindmap_status"] = "error"
             response["mindmap"] = None

requirements.txt

@@ -3,3 +3,4 @@ gradio==5.8.0
 python-dotenv==1.0.1
 pdfplumber==0.11.4
 requests==2.32.3
+g4f[all]==0.4.0.2