Update app.py

app.py

@@ -154,6 +154,75 @@ def google_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_
     print(f"Total results fetched: {len(all_results)}")
     return all_results
 
+def google_news_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_verify=None):
+    print(f"Searching Google News for term: {term}")
+    escaped_term = urllib.parse.quote_plus(term)
+    start = 0
+    all_results = []
+
+    with requests.Session() as session:
+        while len(all_results) < num_results:
+            try:
+                user_agent = random.choice(_useragent_list)
+                headers = {
+                    'User-Agent': user_agent
+                }
+                print(f"Using User-Agent: {headers['User-Agent']}")
+
+                resp = session.get(
+                    url="https://news.google.com/search",
+                    headers=headers,
+                    params={
+                        "q": term,
+                        "hl": lang,
+                        "gl": "US",  # You can change this to target a specific country
+                        "ceid": "US:en"  # Change this according to your language and country
+                    },
+                    timeout=timeout,
+                    verify=ssl_verify,
+                )
+                resp.raise_for_status()
+            except requests.exceptions.RequestException as e:
+                print(f"Error fetching search results: {e}")
+                break
+
+            soup = BeautifulSoup(resp.text, "html.parser")
+            articles = soup.find_all("article")
+            
+            for article in articles:
+                if len(all_results) >= num_results:
+                    break
+                
+                link_element = article.find("a", class_="VDXfz")
+                if link_element:
+                    # Google News uses relative URLs, so we need to construct the full URL
+                    relative_link = link_element['href']
+                    full_link = f"https://news.google.com{relative_link[1:]}"  # Remove the leading '.'
+                    
+                    title = link_element.text
+                    
+                    try:
+                        # Fetch the actual article
+                        article_page = session.get(full_link, headers=headers, timeout=timeout)
+                        article_page.raise_for_status()
+                        article_content = extract_text_from_webpage(article_page.text)
+                        
+                        all_results.append({"link": full_link, "title": title, "text": article_content})
+                    except requests.exceptions.RequestException as e:
+                        print(f"Error fetching or processing {full_link}: {e}")
+                        all_results.append({"link": full_link, "title": title, "text": None})
+                else:
+                    print("No link found in article.")
+
+            if len(articles) == 0:
+                print("No more results found.")
+                break
+
+            start += len(articles)
+
+    print(f"Total news results fetched: {len(all_results)}")
+    return all_results
+
 def summarize_webpage(url, content, query, instructions, max_chars=1000):
     # Preprocess the content
     preprocessed_text = preprocess_web_content(content, query.split())
@@ -409,28 +478,38 @@ def save_text_to_pdf(text, output_path):
     print("PDF saved successfully.")
 
 # Integrated function to perform web scraping, formatting, and text generation
-def scrape_and_display(query, num_results, instructions, web_search=True, temperature=0.7, repetition_penalty=1.0, top_p=0.9):
+def scrape_and_display(query, num_results, instructions, web_search=True, use_news=False, temperature=0.7, repetition_penalty=1.0, top_p=0.9):
     print(f"Scraping and displaying results for query: {query} with num_results: {num_results}")
     if web_search:
-        search_results = google_search(query, num_results, instructions=instructions)
-        formatted_prompt = format_prompt(query, search_results, instructions)
+        if use_news:
+            search_results = google_news_search(query, num_results)
+        else:
+            search_results = google_search(query, num_results, instructions=instructions)
+        
+        # Summarize each result
+        summarized_results = []
+        for result in search_results:
+            summary = summarize_webpage(result['link'], result['text'], query, instructions)
+            summarized_results.append({"link": result['link'], "text": summary})
+        
+        formatted_prompt = format_prompt(query, summarized_results, instructions)
         generated_summary = generate_text(formatted_prompt, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
     else:
         formatted_prompt = format_prompt_with_instructions(query, instructions)
         generated_summary = generate_text(formatted_prompt, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
+    
     print("Scraping and display complete.")
     if generated_summary:
-        # Extract and return text starting from "Assistant:"
         assistant_index = generated_summary.find("Assistant:")
         if assistant_index != -1:
             generated_summary = generated_summary[assistant_index:]
         else:
             generated_summary = "Assistant: No response generated."
-    print(f"Generated summary: {generated_summary}")  # Debugging line
+    print(f"Generated summary: {generated_summary}")
     return generated_summary
 
 # Main Gradio interface function
-def gradio_interface(query, use_dashboard, use_pdf, pdf, num_results, custom_instructions, temperature, repetition_penalty, top_p, clear_cache_flag):
+def gradio_interface(query, use_dashboard, use_news, use_pdf, pdf, num_results, custom_instructions, temperature, repetition_penalty, top_p, clear_cache_flag):
     if clear_cache_flag:
         return clear_cache()
     
@@ -439,26 +518,27 @@ def gradio_interface(query, use_dashboard, use_pdf, pdf, num_results, custom_ins
         for query_type, query_info in PREDEFINED_QUERIES.items():
             formatted_query = query_info['query'].format(company=query)
             formatted_instructions = query_info['instructions'].format(company=query)
-            result = scrape_and_display(formatted_query, num_results=num_results, instructions=formatted_instructions, web_search=True, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
+            result = scrape_and_display(formatted_query, num_results=num_results, instructions=formatted_instructions, web_search=True, use_news=(query_type == "Recent News"), temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
             results.append(f"**{query_type}**\n\n{result}\n\n")
         generated_summary = "\n".join(results)
     elif use_pdf and pdf is not None:
         pdf_text = read_pdf(pdf)
         generated_summary = scrape_and_display(pdf_text, num_results=0, instructions=custom_instructions, web_search=False, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
     else:
-        generated_summary = scrape_and_display(query, num_results=num_results, instructions=custom_instructions, web_search=True, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
+        generated_summary = scrape_and_display(query, num_results=num_results, instructions=custom_instructions, web_search=True, use_news=use_news, temperature=temperature, repetition_penalty=repetition_penalty, top_p=top_p)
     
     output_pdf_path = "output_summary.pdf"
     save_text_to_pdf(generated_summary, output_pdf_path)
     
     return generated_summary, output_pdf_path
 
-# Deploy Gradio Interface
+# Update the Gradio Interface
 gr.Interface(
     fn=gradio_interface,
     inputs=[
         gr.Textbox(label="Company Name or Query"),
         gr.Checkbox(label="Use Dashboard"),
+        gr.Checkbox(label="Use News Search"),  # New checkbox for news search
         gr.Checkbox(label="Use PDF"),
         gr.File(label="Upload PDF"),
         gr.Slider(minimum=1, maximum=20, value=5, step=1, label="Number of Results"),
@@ -470,6 +550,6 @@ gr.Interface(
     ],
     outputs=["text", gr.File(label="Generated PDF")],
     title="Financial Analyst AI Assistant",
-    description="Enter a company name to get a financial dashboard, or enter a custom query. Optionally, upload a PDF for analysis. Adjust parameters as needed for optimal results.",
+    description="Enter a company name to get a financial dashboard, or enter a custom query. Use the news search option for recent articles. Optionally, upload a PDF for analysis. Adjust parameters as needed for optimal results.",
     allow_flagging="never"
 ).launch(share=True)