Update app.py

app.py

@@ -22,6 +22,8 @@ from dotenv import load_dotenv
 import certifi
 from bs4 import BeautifulSoup
 import requests
+from trafilatura.settings import use_config
+from urllib.request import urlopen, Request
 
 # Load environment variables from a .env file
 load_dotenv()
@@ -253,7 +255,7 @@ Remember to focus on financial aspects and implications in your assessment and s
         logger.error(f"Error assessing relevance and summarizing with LLM: {e}")
         return "Error: Unable to assess relevance and summarize"
 
-def scrape_full_content(url, scraper="trafilatura", max_chars=3000, timeout=5):
+def scrape_full_content(url, scraper="bs4", max_chars=3000, timeout=5):
     try:
         logger.info(f"Scraping full content from: {url}")
         
@@ -319,12 +321,7 @@ Your response should be detailed, informative, accurate, and directly relevant t
         logger.error(f"Error in LLM summarization: {e}")
         return "Error: Unable to generate a summary. Please try again."
 
-import requests
-from trafilatura import extract
-from trafilatura.settings import use_config
-from urllib.request import urlopen, Request
-
-def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura", max_chars=3000, time_range="", language="all", category="", 
+def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_chars=3000, time_range="", language="all", category="", 
                       engines=[], safesearch=2, method="GET", llm_temperature=0.2, timeout=5):
     try:
         # Step 1: Rephrase the Query
@@ -561,7 +558,7 @@ iface = gr.ChatInterface(
     description="Enter your query, and I'll search the web for the most recent and relevant financial news, scrape content, and provide summarized results.",
     additional_inputs=[
         gr.Slider(5, 20, value=10, step=1, label="Number of initial results"),
-        gr.Dropdown(["bs4", "trafilatura"], value="trafilatura", label="Scraping Method"),
+        gr.Dropdown(["bs4", "trafilatura"], value="bs4", label="Scraping Method"),
         gr.Slider(500, 10000, value=1500, step=100, label="Max characters to retrieve"),
         gr.Dropdown(["", "day", "week", "month", "year"], value="year", label="Time Range"),
         gr.Dropdown(["all", "en", "fr", "de", "es", "it", "nl", "pt", "pl", "ru", "zh"], value="en", label="Language"),