Update twitter_service.py

twitter_service.py

@@ -1,7 +1,9 @@
 import asyncio
 import json
 import os
+import re
 import time
+import xml.etree.ElementTree as ET
 from datetime import datetime, timedelta, timezone
 from typing import Dict, List, Optional
 
@@ -13,7 +15,7 @@ from models import NewsSource, Tweet
 
 
 class RssTwitterService:
-    """Service for collecting tweets via RSS feeds."""
+    """Service for collecting tweets via RSSHub's XML feeds."""
 
     def __init__(self):
         self.cache_expiry = int(os.getenv("CACHE_EXPIRY_MINUTES", 120))
@@ -35,14 +37,16 @@ class RssTwitterService:
         }
         self.last_stats_reset = time.time()
         
-        # Map of Twitter handles to RSS feed URLs - directly added
+        # Map of Twitter handles to RSSHub URLs - directly added
+        self.rsshub_base_url = "https://rsshub.app/twitter/user/"
         self.rss_feed_urls = {
-            "sidhant": "https://rsshub.app/twitter/user/sidhant",
-            "ShivAroor": "https://rsshub.app/twitter/user/ShivAroor",
-            "IAF_MCC": "https://rsshub.app/twitter/user/IAF_MCC",
-            "adgpi": "https://rsshub.app/twitter/user/adgpi",
-            "SpokespersonMoD": "https://rsshub.app/twitter/user/SpokespersonMoD",
-            "MIB_India": "https://rsshub.app/twitter/user/MIB_India",
+            "sidhant": f"{self.rsshub_base_url}sidhant",
+            "ShivAroor": f"{self.rsshub_base_url}ShivAroor",
+            "IAF_MCC": f"{self.rsshub_base_url}IAF_MCC",
+            "adgpi": f"{self.rsshub_base_url}adgpi",
+            "SpokespersonMoD": f"{self.rsshub_base_url}SpokespersonMoD",
+            "MIB_India": f"{self.rsshub_base_url}MIB_India",
+            "DrSJaishankar": f"{self.rsshub_base_url}DrSJaishankar"
         }
         
         # Default trusted news sources
@@ -59,7 +63,7 @@ class RssTwitterService:
     async def initialize(self) -> bool:
         """Initialize the Twitter service with RSS feed capability."""
         try:
-            logger.info("Initializing Twitter service with RSS feed capability")
+            logger.info("Initializing Twitter service with RSSHub XML feed capability")
             
             # Initialize HTTP client
             self.client = httpx.AsyncClient(
@@ -74,11 +78,11 @@ class RssTwitterService:
             # Schedule background maintenance
             asyncio.create_task(self._background_maintenance())
             
-            logger.info("RSS Twitter service initialized successfully")
+            logger.info("RSSHub Twitter service initialized successfully")
             return True
             
         except Exception as e:
-            logger.error(f"Failed to initialize RSS Twitter service: {str(e)}")
+            logger.error(f"Failed to initialize RSSHub Twitter service: {str(e)}")
             return False
 
     async def _background_maintenance(self):
@@ -108,7 +112,7 @@ class RssTwitterService:
         cache_hit_rate = self.stats["cache_hits"] / total_requests * 100
         error_rate = self.stats["errors"] / total_requests * 100
         
-        logger.info(f"RssTwitterService stats - Requests: {total_requests}, " +
+        logger.info(f"RssHubTwitterService stats - Requests: {total_requests}, " +
                     f"Cache hits: {self.stats['cache_hits']} ({cache_hit_rate:.1f}%), " +
                     f"Errors: {self.stats['errors']} ({error_rate:.1f}%)")
     
@@ -136,12 +140,11 @@ class RssTwitterService:
 
     def _get_cache_key(self, twitter_handle, limit):
         """Generate a cache key for a specific Twitter source"""
-        return f"{twitter_handle}_{limit}"
+        return f"{twitter_handle}_{limit}_rsshub"
 
     def _get_cache_path(self, cache_key):
         """Get filesystem path for a cache key"""
         # Create a safe filename from the cache key
-        import re
         safe_key = re.sub(r'[^a-zA-Z0-9_-]', '_', cache_key)
         return os.path.join(self.tweet_cache_dir, f"{safe_key}.json")
 
@@ -208,7 +211,8 @@ class RssTwitterService:
                 'author': tweet.author,
                 'created_at': created_at.isoformat() if hasattr(created_at, 'isoformat') else str(created_at),
                 'engagement': tweet.engagement,
-                'url': tweet.url
+                'url': tweet.url,
+                'image_url': getattr(tweet, 'image_url', None)  # Handle optional image_url
             })
         
         # Save to disk cache
@@ -222,18 +226,19 @@ class RssTwitterService:
         except Exception as e:
             logger.error(f"Error writing to cache file {cache_path}: {e}")
 
-    def register_rss_feed(self, twitter_handle: str, rss_url: str):
-        """Register an RSS feed URL for a specific Twitter handle"""
-        self.rss_feed_urls[twitter_handle] = rss_url
-        logger.info(f"Registered RSS feed for {twitter_handle}: {rss_url}")
+    def register_rss_feed(self, twitter_handle: str):
+        """Register an RSSHub feed URL for a specific Twitter handle"""
+        self.rss_feed_urls[twitter_handle] = f"{self.rsshub_base_url}{twitter_handle}"
+        logger.info(f"Registered RSSHub feed for {twitter_handle}")
     
-    def register_rss_feed_batch(self, feed_map: Dict[str, str]):
-        """Register multiple RSS feeds at once"""
-        self.rss_feed_urls.update(feed_map)
-        logger.info(f"Registered {len(feed_map)} RSS feeds")
+    def register_rss_feed_batch(self, twitter_handles: List[str]):
+        """Register multiple RSSHub feeds at once"""
+        for handle in twitter_handles:
+            self.rss_feed_urls[handle] = f"{self.rsshub_base_url}{handle}"
+        logger.info(f"Registered {len(twitter_handles)} RSSHub feeds")
             
     async def get_tweets_from_source(self, source: NewsSource, limit: int = 20, retries: int = 3) -> List[Tweet]:
-        """Get tweets from a specific Twitter source using RSS feed."""
+        """Get tweets from a specific Twitter source using RSSHub XML feed."""
         cache_key = self._get_cache_key(source.twitter_handle, limit)
         
         # Check cache first
@@ -247,86 +252,103 @@ class RssTwitterService:
         # Check if we have a registered RSS feed for this Twitter handle
         rss_url = self.rss_feed_urls.get(source.twitter_handle)
         if not rss_url:
-            logger.warning(f"No RSS feed registered for {source.twitter_handle}")
-            return []
+            # Auto-register the feed if not already registered
+            rss_url = f"{self.rsshub_base_url}{source.twitter_handle}"
+            self.rss_feed_urls[source.twitter_handle] = rss_url
+            logger.info(f"Auto-registered RSSHub feed for {source.twitter_handle}")
         
         # Extract tweets with retry logic
         tweets = []
         
         for attempt in range(retries + 1):
             try:
-                logger.info(f"Fetching tweets from RSS feed for {source.twitter_handle} (attempt {attempt + 1}/{retries + 1})")
+                logger.info(f"Fetching tweets from RSSHub for {source.twitter_handle} (attempt {attempt + 1}/{retries + 1})")
                 
-                # Add cache-busting parameter
+                # Add cache-busting parameter to avoid RSSHub's cache
                 params = {"_": str(int(time.time()))}
                 
                 response = await self.client.get(rss_url, params=params)
                 
                 if response.status_code == 200:
-                    # Parse the RSS JSON feed
+                    # Parse the RSS XML feed
                     try:
                         self.stats["success"] += 1
-                        rss_data = response.json()
                         
-                        # Parse items from the feed
-                        if "items" in rss_data:
-                            items = rss_data["items"][:limit]
-                            
-                            for item in items:
+                        # Parse XML response
+                        root = ET.fromstring(response.text)
+                        
+                        # Find all item elements (tweets)
+                        ns = {'atom': 'http://www.w3.org/2005/Atom'}  # Define namespace if needed
+                        items = root.findall('.//item')[:limit]  # Limit to specified number
+                        
+                        for item in items:
+                            try:
+                                # Extract tweet details from XML
+                                title_elem = item.find('title')
+                                title = title_elem.text if title_elem is not None else ""
+                                
+                                description_elem = item.find('description')
+                                description = description_elem.text if description_elem is not None else ""
+                                
+                                link_elem = item.find('link')
+                                url = link_elem.text if link_elem is not None else ""
+                                
+                                # Extract tweet ID from URL
+                                tweet_id = url.split("/status/")[-1] if "/status/" in url else ""
+                                
+                                # Get the timestamp
+                                pub_date_elem = item.find('pubDate')
+                                date_str = pub_date_elem.text if pub_date_elem is not None else ""
+                                
                                 try:
-                                    # Extract tweet ID from the URL
-                                    tweet_id = item.get("id", "").split("/")[-1]
-                                    if not tweet_id:
-                                        continue
-                                    
-                                    # Extract tweet text
-                                    tweet_text = item.get("content_text", item.get("title", ""))
-                                    
-                                    # Extract timestamp
-                                    date_str = item.get("date_published", "")
-                                    try:
-                                        # Convert to datetime then strip timezone info to make naive
-                                        dt = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
-                                        created_at = dt.replace(tzinfo=None)
-                                    except (ValueError, TypeError):
-                                        created_at = datetime.now()
-                                    
-                                    # Extract engagement metrics if available
-                                    engagement = {"likes": 0, "retweets": 0, "replies": 0, "views": 0}
-                                    
-                                    # Try to extract engagement from extensions or additional fields
-                                    if "x_metadata" in item:
-                                        x_data = item["x_metadata"]
-                                        engagement["likes"] = x_data.get("likes", 0)
-                                        engagement["retweets"] = x_data.get("retweets", 0)
-                                        engagement["replies"] = x_data.get("replies", 0)
-                                        engagement["views"] = x_data.get("views", 0)
-                                    
-                                    # Construct tweet URL
-                                    tweet_url = item.get("url", f"https://x.com/{source.twitter_handle}/status/{tweet_id}")
-                                    
-                                    tweets.append(
-                                        Tweet(
-                                            id=tweet_id,
-                                            text=tweet_text,
-                                            author=source.twitter_handle,
-                                            created_at=created_at,
-                                            engagement=engagement,
-                                            url=tweet_url
-                                        )
+                                    # Parse RSS date format
+                                    created_at = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z")
+                                except (ValueError, TypeError):
+                                    logger.warning(f"Date parsing error for {source.twitter_handle}: {date_str}")
+                                    created_at = datetime.now()
+                                
+                                # Get author
+                                author_elem = item.find('author')
+                                author = author_elem.text if author_elem is not None else source.twitter_handle
+                                
+                                # Extract image URL from description
+                                image_url = None
+                                if description:
+                                    # Try to find image in description
+                                    img_match = re.search(r'src="([^"]+)"', description)
+                                    if img_match:
+                                        image_url = img_match.group(1)
+                                
+                                # Extract text content
+                                # For the text, use title as it's cleaner than description
+                                tweet_text = title
+                                
+                                # Mock engagement metrics (not provided in RSS)
+                                engagement = {"likes": 0, "retweets": 0, "replies": 0, "views": 0}
+                                
+                                tweets.append(
+                                    Tweet(
+                                        id=tweet_id,
+                                        text=tweet_text,
+                                        author=source.twitter_handle,
+                                        created_at=created_at,
+                                        engagement=engagement,
+                                        url=url,
+                                        image_url=image_url
                                     )
-                                except Exception as e:
-                                    logger.error(f"Error processing tweet from RSS for {source.twitter_handle}: {str(e)}")
+                                )
+                            except Exception as e:
+                                logger.error(f"Error processing tweet from RSSHub for {source.twitter_handle}: {str(e)}")
                         
                         # Cache the results
                         if tweets:
                             self._save_to_cache(cache_key, tweets)
-                            logger.info(f"Fetched and cached {len(tweets)} tweets from RSS feed for {source.twitter_handle}")
+                            logger.info(f"Fetched and cached {len(tweets)} tweets from RSSHub for {source.twitter_handle}")
                             
                         return tweets
                         
-                    except json.JSONDecodeError:
-                        logger.error(f"Invalid JSON response from RSS feed for {source.twitter_handle}")
+                    except Exception as e:
+                        logger.error(f"Error parsing XML from RSSHub for {source.twitter_handle}: {str(e)}")
                         self.stats["errors"] += 1
                         
                         if attempt < retries:
@@ -337,7 +359,7 @@ class RssTwitterService:
                 else:
                     # HTTP error
                     self.stats["errors"] += 1
-                    logger.error(f"Failed to fetch RSS feed for {source.twitter_handle}: HTTP {response.status_code}")
+                    logger.error(f"Failed to fetch RSSHub feed for {source.twitter_handle}: HTTP {response.status_code}")
                     
                     if attempt < retries:
                         await asyncio.sleep(5)
@@ -347,7 +369,7 @@ class RssTwitterService:
                 
             except Exception as e:
                 self.stats["errors"] += 1
-                logger.error(f"Error fetching RSS feed for {source.twitter_handle}: {str(e)}")
+                logger.error(f"Error fetching RSSHub feed for {source.twitter_handle}: {str(e)}")
                 
                 if attempt < retries:
                     await asyncio.sleep(5)
@@ -363,14 +385,13 @@ class RssTwitterService:
         # Create naive datetime for consistent comparisons
         cutoff_date = datetime.now() - timedelta(days=days_back)
         
-        # Filter to active sources that have RSS feeds
-        active_sources = [source for source in self.news_sources 
-                          if source.is_active and source.twitter_handle in self.rss_feed_urls]
+        # Filter to active sources
+        active_sources = [source for source in self.news_sources if source.is_active]
         
         # Sort sources by reliability score (prioritize higher scores)
         active_sources.sort(key=lambda s: s.reliability_score, reverse=True)
         
-        logger.info(f"Collecting tweets from {len(active_sources)} trusted news sources with RSS feeds")
+        logger.info(f"Collecting tweets from {len(active_sources)} trusted news sources with RSSHub")
         
         # Process sources in parallel
         tasks = []