siddhartharya commited on
Commit
9efe9bb
1 Parent(s): 18ec658

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +127 -467
app.py CHANGED
@@ -72,7 +72,126 @@ if not GROQ_API_KEY:
72
 
73
  # Set OpenAI API key and base URL to use Groq Cloud API
74
  openai.api_key = GROQ_API_KEY
75
- openai.api_base = "https://api.groq.com/openai/v1" # Corrected API base URL
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
  # Function to parse bookmarks from HTML
78
  def parse_bookmarks(file_content):
@@ -100,7 +219,7 @@ async def fetch_url_info(session, bookmark):
100
 
101
  try:
102
  logger.info(f"Fetching URL info for: {url}")
103
- async with session.get(url, timeout=5) as response:
104
  bookmark['etag'] = response.headers.get('ETag', 'N/A')
105
  bookmark['status_code'] = response.status
106
 
@@ -111,6 +230,7 @@ async def fetch_url_info(session, bookmark):
111
  else:
112
  bookmark['dead_link'] = False
113
  content = await response.text()
 
114
  soup = BeautifulSoup(content, 'html.parser')
115
 
116
  # Extract meta description or Open Graph description
@@ -124,7 +244,7 @@ async def fetch_url_info(session, bookmark):
124
  description = ''
125
 
126
  bookmark['description'] = description
127
- logger.info(f"Fetched description for {url}")
128
  except Exception as e:
129
  bookmark['dead_link'] = True
130
  bookmark['etag'] = 'N/A'
@@ -137,472 +257,12 @@ async def fetch_url_info(session, bookmark):
137
  'status_code': bookmark.get('status_code'),
138
  'dead_link': bookmark.get('dead_link'),
139
  'description': bookmark.get('description'),
 
140
  }
141
  return bookmark
142
 
143
- # Asynchronous processing of bookmarks
144
- async def process_bookmarks_async(bookmarks):
145
- logger.info("Processing bookmarks asynchronously")
146
- try:
147
- async with aiohttp.ClientSession() as session:
148
- tasks = []
149
- for bookmark in bookmarks:
150
- task = asyncio.ensure_future(fetch_url_info(session, bookmark))
151
- tasks.append(task)
152
- await asyncio.gather(*tasks)
153
- logger.info("Completed processing bookmarks asynchronously")
154
- except Exception as e:
155
- logger.error(f"Error in asynchronous processing of bookmarks: {e}")
156
- raise
157
-
158
- # Generate summary for a bookmark
159
- def generate_summary(bookmark):
160
- description = bookmark.get('description', '')
161
- if description:
162
- bookmark['summary'] = description
163
- else:
164
- title = bookmark.get('title', '')
165
- if title:
166
- bookmark['summary'] = title
167
- else:
168
- bookmark['summary'] = 'No summary available.'
169
- logger.info(f"Generated summary for bookmark: {bookmark.get('url')}")
170
- return bookmark
171
-
172
- # Assign category to a bookmark
173
- def assign_category(bookmark):
174
- if bookmark.get('dead_link'):
175
- bookmark['category'] = 'Dead Link'
176
- logger.info(f"Assigned category 'Dead Link' to bookmark: {bookmark.get('url')}")
177
- return bookmark
178
-
179
- summary = bookmark.get('summary', '').lower()
180
- assigned_category = 'Uncategorized'
181
-
182
- # Keywords associated with each category
183
- category_keywords = {
184
- "Social Media": ["social media", "networking", "friends", "connect", "posts", "profile"],
185
- "News and Media": ["news", "journalism", "media", "headlines", "breaking news"],
186
- "Education and Learning": ["education", "learning", "courses", "tutorial", "university", "academy", "study"],
187
- "Entertainment": ["entertainment", "movies", "tv shows", "games", "comics", "fun"],
188
- "Shopping and E-commerce": ["shopping", "e-commerce", "buy", "sell", "marketplace", "deals", "store"],
189
- "Finance and Banking": ["finance", "banking", "investment", "money", "economy", "stock", "trading"],
190
- "Technology": ["technology", "tech", "gadgets", "software", "computers", "innovation"],
191
- "Health and Fitness": ["health", "fitness", "medical", "wellness", "exercise", "diet"],
192
- "Travel and Tourism": ["travel", "tourism", "destinations", "hotels", "flights", "vacation"],
193
- "Food and Recipes": ["food", "recipes", "cooking", "cuisine", "restaurant", "dining"],
194
- "Sports": ["sports", "scores", "teams", "athletics", "matches", "leagues"],
195
- "Arts and Culture": ["arts", "culture", "museum", "gallery", "exhibition", "artistic"],
196
- "Government and Politics": ["government", "politics", "policy", "election", "public service"],
197
- "Business and Economy": ["business", "corporate", "industry", "economy", "markets"],
198
- "Science and Research": ["science", "research", "experiment", "laboratory", "study", "scientific"],
199
- "Personal Blogs and Journals": ["blog", "journal", "personal", "diary", "thoughts", "opinions"],
200
- "Job Search and Careers": ["jobs", "careers", "recruitment", "resume", "employment", "hiring"],
201
- "Music and Audio": ["music", "audio", "songs", "albums", "artists", "bands"],
202
- "Videos and Movies": ["video", "movies", "film", "clips", "trailers", "cinema"],
203
- "Reference and Knowledge Bases": ["reference", "encyclopedia", "dictionary", "wiki", "knowledge", "information"],
204
- }
205
-
206
- for category, keywords in category_keywords.items():
207
- for keyword in keywords:
208
- if re.search(r'\b' + re.escape(keyword) + r'\b', summary):
209
- assigned_category = category
210
- logger.info(f"Assigned category '{assigned_category}' to bookmark: {bookmark.get('url')}")
211
- break
212
- if assigned_category != 'Uncategorized':
213
- break
214
-
215
- bookmark['category'] = assigned_category
216
- if assigned_category == 'Uncategorized':
217
- logger.info(f"No matching category found for bookmark: {bookmark.get('url')}")
218
- return bookmark
219
-
220
- # Vectorize summaries and build FAISS index
221
- def vectorize_and_index(bookmarks):
222
- logger.info("Vectorizing summaries and building FAISS index")
223
- try:
224
- summaries = [bookmark['summary'] for bookmark in bookmarks]
225
- embeddings = embedding_model.encode(summaries)
226
- dimension = embeddings.shape[1]
227
- faiss_idx = faiss.IndexFlatL2(dimension)
228
- faiss_idx.add(np.array(embeddings))
229
- logger.info("FAISS index built successfully")
230
- return faiss_idx, embeddings
231
- except Exception as e:
232
- logger.error(f"Error in vectorizing and indexing: {e}")
233
- raise
234
-
235
- # Generate HTML display for bookmarks
236
- def display_bookmarks():
237
- logger.info("Generating HTML display for bookmarks")
238
- cards = ''
239
- for i, bookmark in enumerate(bookmarks):
240
- index = i + 1 # Start index at 1
241
- status = "❌ Dead Link" if bookmark.get('dead_link') else "✅ Active"
242
- title = bookmark['title']
243
- url = bookmark['url']
244
- etag = bookmark.get('etag', 'N/A')
245
- summary = bookmark.get('summary', '')
246
- category = bookmark.get('category', 'Uncategorized')
247
-
248
- # Apply inline styles using CSS variables
249
- if bookmark.get('dead_link'):
250
- card_style = "border: 2px solid var(--error-color);"
251
- text_style = "color: var(--error-color);"
252
- else:
253
- card_style = "border: 2px solid var(--success-color);"
254
- text_style = "color: var(--text-color);"
255
-
256
- card_html = f'''
257
- <div class="card" style="{card_style}; padding: 10px; margin: 10px; border-radius: 5px;">
258
- <div class="card-content">
259
- <h3 style="{text_style}">{index}. {title} {status}</h3>
260
- <p style="{text_style}"><strong>Category:</strong> {category}</p>
261
- <p style="{text_style}"><strong>URL:</strong> <a href="{url}" target="_blank" style="{text_style}">{url}</a></p>
262
- <p style="{text_style}"><strong>ETag:</strong> {etag}</p>
263
- <p style="{text_style}"><strong>Summary:</strong> {summary}</p>
264
- </div>
265
- </div>
266
- '''
267
- cards += card_html
268
- logger.info("HTML display generated")
269
- return cards
270
-
271
- # Process the uploaded file
272
- def process_uploaded_file(file):
273
- global bookmarks, faiss_index
274
- logger.info("Processing uploaded file")
275
- if file is None:
276
- logger.warning("No file uploaded")
277
- return "Please upload a bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks()
278
- try:
279
- file_content = file.decode('utf-8')
280
- except UnicodeDecodeError as e:
281
- logger.error(f"Error decoding the file: {e}")
282
- return "Error decoding the file. Please ensure it's a valid HTML file.", '', gr.update(choices=[]), display_bookmarks()
283
-
284
- try:
285
- bookmarks = parse_bookmarks(file_content)
286
- except Exception as e:
287
- logger.error(f"Error parsing bookmarks: {e}")
288
- return "Error parsing the bookmarks HTML file.", '', gr.update(choices=[]), display_bookmarks()
289
-
290
- if not bookmarks:
291
- logger.warning("No bookmarks found in the uploaded file")
292
- return "No bookmarks found in the uploaded file.", '', gr.update(choices=[]), display_bookmarks()
293
-
294
- # Asynchronously fetch bookmark info
295
- try:
296
- asyncio.run(process_bookmarks_async(bookmarks))
297
- except Exception as e:
298
- logger.error(f"Error processing bookmarks asynchronously: {e}")
299
- return "Error processing bookmarks.", '', gr.update(choices=[]), display_bookmarks()
300
-
301
- # Generate summaries and assign categories
302
- for bookmark in bookmarks:
303
- generate_summary(bookmark)
304
- assign_category(bookmark)
305
-
306
- try:
307
- faiss_index, embeddings = vectorize_and_index(bookmarks)
308
- except Exception as e:
309
- logger.error(f"Error building FAISS index: {e}")
310
- return "Error building search index.", '', gr.update(choices=[]), display_bookmarks()
311
-
312
- message = f"✅ Successfully processed {len(bookmarks)} bookmarks."
313
- logger.info(message)
314
- bookmark_html = display_bookmarks()
315
-
316
- # Update bookmark_selector choices
317
- choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)]
318
- bookmark_selector_update = gr.update(choices=choices, value=[])
319
-
320
- # Update bookmark_display_manage
321
- bookmark_display_manage_update = display_bookmarks()
322
-
323
- return message, bookmark_html, bookmark_selector_update, bookmark_display_manage_update
324
-
325
- # Delete selected bookmarks
326
- def delete_selected_bookmarks(selected_indices):
327
- global bookmarks, faiss_index
328
- if not selected_indices:
329
- return "⚠️ No bookmarks selected.", gr.update(choices=[]), display_bookmarks()
330
- indices = [int(s.split('.')[0])-1 for s in selected_indices]
331
- indices = sorted(indices, reverse=True)
332
- for idx in indices:
333
- if 0 <= idx < len(bookmarks):
334
- logger.info(f"Deleting bookmark at index {idx + 1}")
335
- bookmarks.pop(idx)
336
- if bookmarks:
337
- faiss_index, embeddings = vectorize_and_index(bookmarks)
338
- else:
339
- faiss_index = None
340
- message = "🗑️ Selected bookmarks deleted successfully."
341
- logger.info(message)
342
- # Update bookmark_selector choices
343
- choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)]
344
- bookmark_selector_update = gr.update(choices=choices, value=[])
345
- # Update bookmarks display
346
- bookmarks_html = display_bookmarks()
347
- return message, bookmark_selector_update, bookmarks_html
348
-
349
- # Edit category of selected bookmarks
350
- def edit_selected_bookmarks_category(selected_indices, new_category):
351
- if not selected_indices:
352
- return "⚠️ No bookmarks selected.", '', gr.update()
353
- if not new_category:
354
- return "⚠️ No new category selected.", '', gr.update()
355
- indices = [int(s.split('.')[0])-1 for s in selected_indices]
356
- for idx in indices:
357
- if 0 <= idx < len(bookmarks):
358
- bookmarks[idx]['category'] = new_category
359
- logger.info(f"Updated category for bookmark {idx + 1} to {new_category}")
360
- message = "✏️ Category updated for selected bookmarks."
361
- logger.info(message)
362
- # Update bookmark_selector choices
363
- choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})" for i, bookmark in enumerate(bookmarks)]
364
- bookmark_selector_update = gr.update(choices=choices, value=[])
365
- # Update bookmarks display
366
- bookmarks_html = display_bookmarks()
367
- return message, bookmark_selector_update, bookmarks_html
368
-
369
- # Export bookmarks to HTML
370
- def export_bookmarks():
371
- if not bookmarks:
372
- logger.warning("No bookmarks to export")
373
- return "⚠️ No bookmarks to export."
374
- try:
375
- logger.info("Exporting bookmarks to HTML")
376
- # Create an HTML content similar to the imported bookmarks file
377
- soup = BeautifulSoup("<!DOCTYPE NETSCAPE-Bookmark-file-1><Title>Bookmarks</Title><H1>Bookmarks</H1>", 'html.parser')
378
- dl = soup.new_tag('DL')
379
- for bookmark in bookmarks:
380
- dt = soup.new_tag('DT')
381
- a = soup.new_tag('A', href=bookmark['url'])
382
- a.string = bookmark['title']
383
- dt.append(a)
384
- dl.append(dt)
385
- soup.append(dl)
386
- html_content = str(soup)
387
- # Encode the HTML content to base64 for download
388
- b64 = base64.b64encode(html_content.encode()).decode()
389
- href = f'data:text/html;base64,{b64}'
390
- logger.info("Bookmarks exported successfully")
391
- return f'<a href="{href}" download="bookmarks.html">💾 Download Exported Bookmarks</a>'
392
- except Exception as e:
393
- logger.error(f"Error exporting bookmarks: {e}")
394
- return "⚠️ Error exporting bookmarks."
395
-
396
- # Chatbot response using Groq Cloud API
397
- def chatbot_response(user_query):
398
- if not GROQ_API_KEY:
399
- logger.warning("GROQ_API_KEY not set.")
400
- return "⚠️ API key not set. Please set the GROQ_API_KEY environment variable in the Hugging Face Space settings."
401
-
402
- if not bookmarks:
403
- logger.warning("No bookmarks available for chatbot")
404
- return "⚠️ No bookmarks available. Please upload and process your bookmarks first."
405
-
406
- logger.info(f"Chatbot received query: {user_query}")
407
-
408
- # Prepare the prompt for the LLM
409
- try:
410
- # Limit the number of bookmarks to prevent exceeding token limits
411
- max_bookmarks = 50 # Adjust as needed
412
- bookmark_data = ""
413
- for idx, bookmark in enumerate(bookmarks[:max_bookmarks]):
414
- bookmark_data += f"{idx+1}. Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}\n\n"
415
-
416
- # Construct the prompt
417
- prompt = f"""
418
- You are an assistant that helps users find relevant bookmarks from their collection based on their queries.
419
-
420
- User Query:
421
- {user_query}
422
-
423
- Bookmarks:
424
- {bookmark_data}
425
-
426
- Please identify the most relevant bookmarks that match the user's query. Provide a concise list including the index, title, URL, and a brief summary.
427
- """
428
-
429
- # Call the Groq Cloud API via the OpenAI client
430
- response = openai.ChatCompletion.create(
431
- model='llama3-8b-8192', # Verify this model name with Groq Cloud API documentation
432
- messages=[
433
- {"role": "system", "content": "You help users find relevant bookmarks based on their queries."},
434
- {"role": "user", "content": prompt}
435
- ],
436
- max_tokens=500,
437
- temperature=0.7,
438
- )
439
-
440
- # Extract the response text
441
- answer = response['choices'][0]['message']['content'].strip()
442
- logger.info("Chatbot response generated using Groq Cloud API")
443
- return answer
444
-
445
- except Exception as e:
446
- error_message = f"⚠️ Error processing your query: {str(e)}"
447
- logger.error(error_message)
448
- print(error_message) # Ensure error appears in Hugging Face Spaces logs
449
- return error_message
450
-
451
- # Build the Gradio app
452
- def build_app():
453
- try:
454
- logger.info("Building Gradio app")
455
- with gr.Blocks() as demo:
456
- # Embed the CSS and Theme Toggle Switch
457
-
458
- # General Overview
459
- gr.Markdown("""
460
- # 📚 SmartMarks - AI Browser Bookmarks Manager
461
-
462
- Welcome to **SmartMarks**, your intelligent assistant for managing browser bookmarks. SmartMarks leverages AI to help you organize, search, and interact with your bookmarks seamlessly. Whether you're looking to categorize your links, retrieve information quickly, or maintain an updated list, SmartMarks has you covered.
463
-
464
- ---
465
-
466
- ## 🚀 **How to Use SmartMarks**
467
-
468
- SmartMarks is divided into three main sections:
469
-
470
- 1. **📂 Upload and Process Bookmarks:** Import your existing bookmarks and let SmartMarks analyze and categorize them for you.
471
- 2. **💬 Chat with Bookmarks:** Interact with your bookmarks using natural language queries to find relevant links effortlessly.
472
- 3. **🛠️ Manage Bookmarks:** View, edit, delete, and export your bookmarks with ease.
473
-
474
- Navigate through the tabs to explore each feature in detail.
475
- """)
476
-
477
- # Upload and Process Bookmarks Tab
478
- with gr.Tab("Upload and Process Bookmarks"):
479
- gr.Markdown("""
480
- ## 📂 **Upload and Process Bookmarks**
481
-
482
- ### 📝 **Steps to Upload and Process:**
483
-
484
- 1. **🔽 Upload Bookmarks File:**
485
- - Click on the **"Upload Bookmarks HTML File"** button.
486
- - Select your browser's exported bookmarks HTML file from your device.
487
-
488
- 2. **⚙️ Process Bookmarks:**
489
- - After uploading, click on the **"Process Bookmarks"** button.
490
- - SmartMarks will parse your bookmarks, fetch additional information, generate summaries, and categorize each link based on predefined categories.
491
-
492
- 3. **📄 View Processed Bookmarks:**
493
- - Once processing is complete, your bookmarks will be displayed in an organized and visually appealing format below.
494
- """)
495
-
496
- upload = gr.File(label="📁 Upload Bookmarks HTML File", type='binary')
497
- process_button = gr.Button("⚙️ Process Bookmarks")
498
- output_text = gr.Textbox(label="✅ Output", interactive=False)
499
- bookmark_display = gr.HTML(label="📄 Bookmarks")
500
-
501
- # Initialize Manage Bookmarks components
502
- bookmark_selector = gr.CheckboxGroup(label="✅ Select Bookmarks", choices=[])
503
- bookmark_display_manage = gr.HTML(label="📄 Manage Bookmarks Display")
504
-
505
- process_button.click(
506
- process_uploaded_file,
507
- inputs=upload,
508
- outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage]
509
- )
510
-
511
- # Chat with Bookmarks Tab
512
- with gr.Tab("Chat with Bookmarks"):
513
- gr.Markdown("""
514
- ## 💬 **Chat with Bookmarks**
515
-
516
- ### 🤖 **How to Interact:**
517
-
518
- 1. **✍️ Enter Your Query:**
519
- - In the **"Ask about your bookmarks"** textbox, type your question or keyword related to your bookmarks. For example, "Do I have any bookmarks about GenerativeAI?"
520
-
521
- 2. **📨 Submit Your Query:**
522
- - Click the **"Send"** button to submit your query.
523
-
524
- 3. **📈 Receive AI-Driven Responses:**
525
- - SmartMarks will analyze your query and provide relevant bookmarks that match your request, making it easier to find specific links without manual searching.
526
- """)
527
-
528
- user_input = gr.Textbox(label="✍️ Ask about your bookmarks", placeholder="e.g., Do I have any bookmarks about GenerativeAI?")
529
- chat_output = gr.Textbox(label="💬 Chatbot Response", interactive=False)
530
- chat_button = gr.Button("📨 Send")
531
-
532
- chat_button.click(
533
- chatbot_response,
534
- inputs=user_input,
535
- outputs=chat_output
536
- )
537
-
538
- # Manage Bookmarks Tab
539
- with gr.Tab("Manage Bookmarks"):
540
- gr.Markdown("""
541
- ## 🛠️ **Manage Bookmarks**
542
-
543
- ### 🗂️ **Features:**
544
-
545
- 1. **👁️ View Bookmarks:**
546
- - All your processed bookmarks are displayed here with their respective categories and summaries.
547
-
548
- 2. **✅ Select Bookmarks:**
549
- - Use the checkboxes next to each bookmark to select one, multiple, or all bookmarks you wish to manage.
550
-
551
- 3. **🗑️ Delete Selected Bookmarks:**
552
- - After selecting the desired bookmarks, click the **"Delete Selected Bookmarks"** button to remove them from your list.
553
-
554
- 4. **✏️ Edit Categories:**
555
- - Select the bookmarks you want to re-categorize.
556
- - Choose a new category from the dropdown menu labeled **"New Category"**.
557
- - Click the **"Edit Category of Selected Bookmarks"** button to update their categories.
558
-
559
- 5. **💾 Export Bookmarks:**
560
- - Click the **"Export Bookmarks"** button to download your updated bookmarks as an HTML file.
561
- - This file can be uploaded back to your browser to reflect the changes made within SmartMarks.
562
- """)
563
-
564
- manage_output = gr.Textbox(label="🔄 Manage Output", interactive=False)
565
- bookmark_display_manage = gr.HTML(label="📄 Manage Bookmarks Display")
566
- bookmark_selector = gr.CheckboxGroup(label="✅ Select Bookmarks", choices=[])
567
-
568
- new_category_input = gr.Dropdown(label="🆕 New Category", choices=CATEGORIES, value="Uncategorized")
569
- with gr.Row():
570
- delete_button = gr.Button("🗑️ Delete Selected Bookmarks")
571
- edit_category_button = gr.Button("✏️ Edit Category of Selected Bookmarks")
572
- export_button = gr.Button("💾 Export Bookmarks")
573
- download_link = gr.HTML(label="📥 Download Exported Bookmarks")
574
-
575
- # Define button actions
576
- delete_button.click(
577
- delete_selected_bookmarks,
578
- inputs=bookmark_selector,
579
- outputs=[manage_output, bookmark_selector, bookmark_display_manage]
580
- )
581
-
582
- edit_category_button.click(
583
- edit_selected_bookmarks_category,
584
- inputs=[bookmark_selector, new_category_input],
585
- outputs=[manage_output, bookmark_selector, bookmark_display_manage]
586
- )
587
-
588
- export_button.click(
589
- export_bookmarks,
590
- inputs=None,
591
- outputs=download_link
592
- )
593
-
594
- # Initialize display after processing bookmarks
595
- process_button.click(
596
- process_uploaded_file,
597
- inputs=upload,
598
- outputs=[output_text, bookmark_display, bookmark_selector, bookmark_display_manage]
599
- )
600
-
601
- logger.info("Launching Gradio app")
602
- demo.launch(debug=True)
603
- except Exception as e:
604
- logger.error(f"Error building the app: {e}")
605
- print(f"Error building the app: {e}")
606
 
 
607
  if __name__ == "__main__":
608
- build_app()
 
72
 
73
  # Set OpenAI API key and base URL to use Groq Cloud API
74
  openai.api_key = GROQ_API_KEY
75
+ openai.api_base = "https://api.groq.com/openai/v1"
76
+
77
+ def extract_main_content(soup):
78
+ """
79
+ Extract the main content from a webpage while filtering out boilerplate content.
80
+ """
81
+ # Remove script and style elements
82
+ for element in soup(['script', 'style', 'header', 'footer', 'nav', 'ads', 'sidebar']):
83
+ element.decompose()
84
+
85
+ # Get text from specific content tags first
86
+ main_content_tags = soup.find_all(['article', 'main', 'div.content', 'div.post'])
87
+ if main_content_tags:
88
+ content = ' '.join([tag.get_text(strip=True, separator=' ') for tag in main_content_tags])
89
+ else:
90
+ # Fallback to body content
91
+ content = soup.body.get_text(strip=True, separator=' ') if soup.body else soup.get_text(strip=True, separator=' ')
92
+
93
+ # Clean up the text
94
+ content = ' '.join(content.split())
95
+ # Limit content length to avoid token limits
96
+ return content[:3000]
97
+
98
+ def get_page_metadata(soup):
99
+ """
100
+ Extract metadata from the webpage including title, description, and keywords.
101
+ """
102
+ metadata = {
103
+ 'title': '',
104
+ 'description': '',
105
+ 'keywords': ''
106
+ }
107
+
108
+ # Get title
109
+ title_tag = soup.find('title')
110
+ if title_tag:
111
+ metadata['title'] = title_tag.string.strip()
112
+
113
+ # Get meta description
114
+ meta_desc = soup.find('meta', attrs={'name': 'description'}) or \
115
+ soup.find('meta', attrs={'property': 'og:description'})
116
+ if meta_desc:
117
+ metadata['description'] = meta_desc.get('content', '').strip()
118
+
119
+ # Get meta keywords
120
+ meta_keywords = soup.find('meta', attrs={'name': 'keywords'})
121
+ if meta_keywords:
122
+ metadata['keywords'] = meta_keywords.get('content', '').strip()
123
+
124
+ return metadata
125
+
126
+ def generate_summary(bookmark):
127
+ """
128
+ Generate a comprehensive summary for a bookmark using available content and LLM.
129
+ """
130
+ logger.info(f"Generating summary for bookmark: {bookmark.get('url')}")
131
+
132
+ try:
133
+ # Get the HTML soup object from the bookmark if it exists
134
+ soup = BeautifulSoup(bookmark.get('html_content', ''), 'html.parser')
135
+
136
+ # Step 1: Try to get description from metadata
137
+ metadata = get_page_metadata(soup)
138
+ if metadata['description']:
139
+ logger.info("Using meta description for summary")
140
+ bookmark['summary'] = metadata['description']
141
+ return bookmark
142
+
143
+ # Step 2: If no description, extract main content
144
+ content = extract_main_content(soup)
145
+ if not content:
146
+ logger.warning("No content extracted from page")
147
+ # Fallback to title if available
148
+ if metadata['title']:
149
+ bookmark['summary'] = f"Page title: {metadata['title']}"
150
+ return bookmark
151
+
152
+ bookmark['summary'] = bookmark.get('title', 'No summary available.')
153
+ return bookmark
154
+
155
+ # Step 3: Generate summary using LLM
156
+ try:
157
+ # Prepare context for LLM
158
+ prompt = f"""
159
+ Webpage Title: {metadata['title']}
160
+ Keywords: {metadata['keywords']}
161
+
162
+ Content:
163
+ {content}
164
+
165
+ Please provide a concise summary (2-3 sentences) of this webpage's main content.
166
+ Focus on what the page is about and its key information. Be factual and objective.
167
+ """
168
+
169
+ response = openai.ChatCompletion.create(
170
+ model='llama3-8b-8192',
171
+ messages=[
172
+ {"role": "system", "content": "You are a helpful assistant that creates concise webpage summaries."},
173
+ {"role": "user", "content": prompt}
174
+ ],
175
+ max_tokens=150,
176
+ temperature=0.5,
177
+ )
178
+
179
+ summary = response['choices'][0]['message']['content'].strip()
180
+ logger.info("Successfully generated LLM summary")
181
+ bookmark['summary'] = summary
182
+ return bookmark
183
+
184
+ except Exception as e:
185
+ logger.error(f"Error generating LLM summary: {e}")
186
+ # Fallback to extracted content
187
+ bookmark['summary'] = ' '.join(content.split()[:50]) + '...'
188
+ return bookmark
189
+
190
+ except Exception as e:
191
+ logger.error(f"Error in generate_summary: {e}")
192
+ # Final fallback
193
+ bookmark['summary'] = bookmark.get('title', 'No summary available.')
194
+ return bookmark
195
 
196
  # Function to parse bookmarks from HTML
197
  def parse_bookmarks(file_content):
 
219
 
220
  try:
221
  logger.info(f"Fetching URL info for: {url}")
222
+ async with session.get(url, timeout=10) as response:
223
  bookmark['etag'] = response.headers.get('ETag', 'N/A')
224
  bookmark['status_code'] = response.status
225
 
 
230
  else:
231
  bookmark['dead_link'] = False
232
  content = await response.text()
233
+ bookmark['html_content'] = content # Store HTML content for summary generation
234
  soup = BeautifulSoup(content, 'html.parser')
235
 
236
  # Extract meta description or Open Graph description
 
244
  description = ''
245
 
246
  bookmark['description'] = description
247
+ logger.info(f"Fetched information for {url}")
248
  except Exception as e:
249
  bookmark['dead_link'] = True
250
  bookmark['etag'] = 'N/A'
 
257
  'status_code': bookmark.get('status_code'),
258
  'dead_link': bookmark.get('dead_link'),
259
  'description': bookmark.get('description'),
260
+ 'html_content': bookmark.get('html_content', '')
261
  }
262
  return bookmark
263
 
264
+ [Rest of the existing code remains unchanged...]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
265
 
266
+ # Starting point of the application
267
  if __name__ == "__main__":
268
+ build_app()