Update app.py

app.py

@@ -155,55 +155,32 @@ def generate_summary(bookmark):
     try:
         html_content = bookmark.get('html_content', '')
 
-        # Check for insufficient or error content
-        error_keywords = ['Access Denied', 'Error', 'Security Check', 'Cloudflare', 'captcha', 'unusual traffic']
-        if not html_content or len(html_content) < 500 or any(keyword.lower() in html_content.lower() for keyword in error_keywords):
-            logger.info(f"Content for {bookmark.get('url')} is insufficient or contains errors. Using prior knowledge.")
-            use_prior_knowledge = True
-        else:
-            use_prior_knowledge = False
-
-        if use_prior_knowledge:
-            # Construct prompt to use prior knowledge
-            prompt = f"""
-You are a knowledgeable assistant.
-
-The user provided a URL: {bookmark.get('url')}
-
-Please provide a concise summary (2-3 sentences) about this website based on your knowledge.
-
-Focus on:
-- The main purpose or topic of the website.
-- Key information or features.
-- Target audience or use case (if apparent).
-
-Be factual and objective.
-"""
-        else:
-            # Get the HTML soup object from the bookmark
-            soup = BeautifulSoup(html_content, 'html.parser')
-
-            # Extract metadata and main content
-            metadata = get_page_metadata(soup)
-            main_content = extract_main_content(soup)
-
-            # Prepare content for the prompt
-            available_content = []
-            if metadata['title']:
-                available_content.append(f"Title: {metadata['title']}")
-            if metadata['description']:
-                available_content.append(f"Description: {metadata['description']}")
-            if metadata['keywords']:
-                available_content.append(f"Keywords: {metadata['keywords']}")
-            if main_content:
-                available_content.append(f"Main Content: {main_content}")
-
-            # Construct the prompt
-            prompt = f"""
+        # Get the HTML soup object from the bookmark
+        soup = BeautifulSoup(html_content, 'html.parser')
+
+        # Extract metadata and main content
+        metadata = get_page_metadata(soup)
+        main_content = extract_main_content(soup)
+
+        # Prepare content for the prompt
+        available_content = []
+        if metadata['title']:
+            available_content.append(f"Title: {metadata['title']}")
+        if metadata['description']:
+            available_content.append(f"Description: {metadata['description']}")
+        if metadata['keywords']:
+            available_content.append(f"Keywords: {metadata['keywords']}")
+        if main_content:
+            available_content.append(f"Main Content: {main_content}")
+
+        # Construct the prompt
+        prompt = f"""
 Analyze and summarize the following webpage content:
 
 {' '.join(available_content)}
 
+If the content seems insufficient or outdated, please use any additional knowledge you have about the current state of the website to provide an accurate summary.
+
 Provide a concise summary (2-3 sentences) focusing on:
 - The main purpose or topic of the page.
 - Key information or features.
@@ -214,7 +191,7 @@ Be factual and objective.
 
         # Call the LLM via Groq Cloud API
         response = openai.ChatCompletion.create(
-            model='llama3-8b-8192',
+            model='llama-3.1-70b-versatile',  # Updated model
             messages=[
                 {"role": "system", "content": "You are a helpful assistant that creates concise webpage summaries."},
                 {"role": "user", "content": prompt}
@@ -269,7 +246,7 @@ async def fetch_url_info(session, bookmark):
                           'Chrome/91.0.4472.124 Safari/537.36',
             'Accept-Language': 'en-US,en;q=0.9',
         }
-        async with session.get(url, timeout=10, headers=headers, allow_redirects=True) as response:
+        async with session.get(url, timeout=20, headers=headers, ssl=False) as response:
             bookmark['etag'] = response.headers.get('ETag', 'N/A')
             bookmark['status_code'] = response.status
 
@@ -353,7 +330,7 @@ Respond with only the category name.
 
     try:
         response = openai.ChatCompletion.create(
-            model='llama3-8b-8192',
+            model='llama-3.1-70b-versatile',  # Updated model
             messages=[
                 {"role": "system", "content": "You categorize webpages based on their content."},
                 {"role": "user", "content": prompt}
@@ -631,7 +608,7 @@ Provide a concise and helpful response.
 """
 
         response = openai.ChatCompletion.create(
-            model='llama3-8b-8192',
+            model='llama-3.1-70b-versatile',  # Updated model
             messages=[
                 {"role": "system", "content": "You assist users by finding relevant information from their bookmarks."},
                 {"role": "user", "content": prompt}