Shreyas094
commited on
Update app.py
Browse files
app.py
CHANGED
@@ -88,7 +88,7 @@ def scrape_with_bs4(url, session, max_chars=None):
|
|
88 |
logger.error(f"Error scraping {url} with BeautifulSoup: {e}")
|
89 |
return ""
|
90 |
|
91 |
-
def scrape_with_trafilatura(url, max_chars=None, timeout=
|
92 |
try:
|
93 |
response = requests.get(url, timeout=timeout)
|
94 |
response.raise_for_status()
|
@@ -234,7 +234,7 @@ Remember to focus on financial aspects and implications in your assessment and s
|
|
234 |
logger.error(f"Error assessing relevance and summarizing with LLM: {e}")
|
235 |
return "Error: Unable to assess relevance and summarize"
|
236 |
|
237 |
-
def scrape_full_content(url, scraper="trafilatura", max_chars=3000, timeout=
|
238 |
try:
|
239 |
logger.info(f"Scraping full content from: {url}")
|
240 |
|
@@ -306,7 +306,7 @@ from trafilatura.settings import use_config
|
|
306 |
from urllib.request import urlopen, Request
|
307 |
|
308 |
def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura", max_chars=3000, time_range="", language="all", category="",
|
309 |
-
engines=[], safesearch=2, method="GET", llm_temperature=0.2, timeout=
|
310 |
try:
|
311 |
# Step 1: Rephrase the Query
|
312 |
rephrased_query = rephrase_query(chat_history, query, temperature=llm_temperature)
|
|
|
88 |
logger.error(f"Error scraping {url} with BeautifulSoup: {e}")
|
89 |
return ""
|
90 |
|
91 |
+
def scrape_with_trafilatura(url, max_chars=None, timeout=5):
|
92 |
try:
|
93 |
response = requests.get(url, timeout=timeout)
|
94 |
response.raise_for_status()
|
|
|
234 |
logger.error(f"Error assessing relevance and summarizing with LLM: {e}")
|
235 |
return "Error: Unable to assess relevance and summarize"
|
236 |
|
237 |
+
def scrape_full_content(url, scraper="trafilatura", max_chars=3000, timeout=5):
|
238 |
try:
|
239 |
logger.info(f"Scraping full content from: {url}")
|
240 |
|
|
|
306 |
from urllib.request import urlopen, Request
|
307 |
|
308 |
def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura", max_chars=3000, time_range="", language="all", category="",
|
309 |
+
engines=[], safesearch=2, method="GET", llm_temperature=0.2, timeout=5):
|
310 |
try:
|
311 |
# Step 1: Rephrase the Query
|
312 |
rephrased_query = rephrase_query(chat_history, query, temperature=llm_temperature)
|