broadfield-dev commited on
Commit
8a41dca
·
verified ·
1 Parent(s): 78dac58

Update rss_processor.py

Browse files
Files changed (1) hide show
  1. rss_processor.py +3 -1
rss_processor.py CHANGED
@@ -77,6 +77,7 @@ def fetch_rss_feeds():
77
  if feed.bozo:
78
  logger.warning(f"Failed to parse {feed_url}: {feed.bozo_exception}")
79
  continue
 
80
  for entry in feed.entries[:5]:
81
  title = entry.get("title", "No Title")
82
  link = entry.get("link", "")
@@ -84,6 +85,7 @@ def fetch_rss_feeds():
84
  article_key = f"{title}|{link}"
85
  if article_key not in seen_articles:
86
  seen_articles.add(article_key)
 
87
  image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or ""
88
  articles.append({
89
  "title": title,
@@ -93,7 +95,7 @@ def fetch_rss_feeds():
93
  "category": categorize_feed(feed_url),
94
  "image": image if image else "",
95
  })
96
- logger.info(f"Processed {len([e for e in feed.entries[:5] if f'{e.get('title', 'No Title')}|{e.get('link', '')}' not in seen_articles])} unique entries from {feed_url}")
97
  except Exception as e:
98
  logger.error(f"Error fetching {feed_url}: {e}")
99
  return articles
 
77
  if feed.bozo:
78
  logger.warning(f"Failed to parse {feed_url}: {feed.bozo_exception}")
79
  continue
80
+ unique_count = 0
81
  for entry in feed.entries[:5]:
82
  title = entry.get("title", "No Title")
83
  link = entry.get("link", "")
 
85
  article_key = f"{title}|{link}"
86
  if article_key not in seen_articles:
87
  seen_articles.add(article_key)
88
+ unique_count += 1
89
  image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or ""
90
  articles.append({
91
  "title": title,
 
95
  "category": categorize_feed(feed_url),
96
  "image": image if image else "",
97
  })
98
+ logger.info(f"Processed {unique_count} unique entries from {feed_url}")
99
  except Exception as e:
100
  logger.error(f"Error fetching {feed_url}: {e}")
101
  return articles