Create rss_processor.py

rss_processor.py

@@ -0,0 +1,278 @@
+import os
+import feedparser
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.docstore.document import Document
+import logging
+from huggingface_hub import HfApi, login
+import shutil
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Constants
+LOCAL_DB_DIR = "chroma_db"
+RSS_FEEDS = [
+    # Academic Papers (Published Papers Across Fields)
+    "https://www.nature.com/nature/current_issue/rss",  # Nature - High-brow science
+    "https://www.science.org/action/showFeed?type=etoc&feed=rss",  # Science journal
+    "https://arxiv.org/rss/cs",  # Computer Science arXiv
+    "https://arxiv.org/rss/physics",  # Physics arXiv
+    "https://arxiv.org/rss/math",  # Mathematics arXiv
+    "https://arxiv.org/rss/astro-ph",  # Astrophysics arXiv
+    "https://arxiv.org/rss/biology",  # Biology arXiv
+    "https://arxiv.org/rss/econ",  # Economics arXiv
+    "https://arxiv.org/rss/stat",  # Statistics arXiv
+    "https://arxiv.org/rss/quant-ph",  # Quantum Physics arXiv
+    "https://www.plos.org/feed/",  # PLOS (Public Library of Science) - multidisciplinary
+    "https://www.journals.uchicago.edu/action/showFeed?type=etoc&feed=rss&jc=pnl",  # Philosophy (e.g., Philosophy of Science)
+    "https://www.annualreviews.org/action/showFeed?type=etoc&feed=rss&jc=anrevo",  # Annual Reviews (various fields)
+    "https://www.jneurosci.org/rss/current.xml",  # Journal of Neuroscience
+    "https://www.cell.com/cell/rss",  # Cell Press - Biology/Medicine
+    "https://www.nejm.org/rss/all-articles.xml",  # New England Journal of Medicine
+    "https://www.lancet.com/rss/lancet_current.xml",  # The Lancet - Medicine
+
+    # Business
+    "https://www.reuters.com/arc/outboundfeeds/business-news/?outputType=rss",  # Reuters Business
+    "https://www.bloomberg.com/feeds/rss/businessweek.xml",  # Bloomberg Businessweek
+    "https://www.ft.com/rss/home",  # Financial Times
+    "https://www.marketwatch.com/rss/topstories",  # MarketWatch Business & Finance
+    "https://www.cnbc.com/id/100003114/device/rss/rss.html",  # CNBC Business News
+    "https://www.foxbusiness.com/rss",  # Fox Business
+    "https://www.wsj.com/xml/rss/3_7085.xml",  # Wall Street Journal - Business & Finance
+    "https://www.bworldonline.com/feed/",  # BusinessWorld Online (Philippines)
+    "https://www.economist.com/business/rss.xml",  # The Economist - Business
+    "https://www.forbes.com/business/feed/",  # Forbes Business
+
+    # Stocks & Markets
+    "https://www.investing.com/rss/news_25.rss",  # Investing.com Stocks & Markets
+    "https://www.cnbc.com/id/100727362/device/rss/rss.html",  # CNBC Market Data
+    "https://www.marketwatch.com/rss/marketpulse",  # MarketWatch Market News
+    "https://www.fool.co.uk/feed/",  # Motley Fool UK - Stock Investing
+    "https://www.zacks.com/rss/zc.xml",  # Zacks Investment Research
+    "https://seekingalpha.com/feed.xml",  # Seeking Alpha - Stock Analysis
+    "https://www.barrons.com/rss.xml",  # Barron's - Market News
+    "https://www.yahoofinance.com/news/rss",  # Yahoo Finance - Stocks & Markets
+
+    # Federal Government
+    "https://www.whitehouse.gov/feed/",  # White House News
+    "https://www.state.gov/rss-feeds/",  # U.S. Department of State
+    "https://www.commerce.gov/feeds/rss",  # U.S. Department of Commerce
+    "https://www.transportation.gov/feeds/rss",  # U.S. Department of Transportation
+    "https://www.ed.gov/rss/",  # U.S. Department of Education
+    "https://www.dol.gov/newsroom/rss-feeds",  # U.S. Department of Labor
+    "https://www.justice.gov/feeds/opa",  # U.S. Department of Justice
+    "https://www.federalreserve.gov/feeds/",  # Federal Reserve Board
+    "https://www.occ.gov/rss-feeds.html",  # Office of the Comptroller of the Currency
+    "https://www.sec.gov/news/sec-rss-feeds.htm",  # SEC News
+    "https://www.bls.gov/rss/",  # U.S. Bureau of Labor Statistics
+    "https://www.usda.gov/rss/",  # U.S. Department of Agriculture
+    "https://www.gao.gov/rss/",  # Government Accountability Office
+    "https://www.cbo.gov/rss/",  # Congressional Budget Office
+    "https://www.fema.gov/rss/",  # Federal Emergency Management Agency
+    "https://www.defense.gov/Newsroom/RSS/",  # U.S. Department of Defense
+    "https://www.hhs.gov/rss/",  # U.S. Department of Health & Human Services
+    "https://www.energy.gov/rss/",  # U.S. Department of Energy
+    "https://www.interior.gov/rss",  # U.S. Department of the Interior
+
+    # Weather
+    "https://www.weather.gov/rss/",  # National Weather Service
+    "https://www.metoffice.gov.uk/weather/warnings-and-advice/uk-warnings/rss",  # UK Met Office
+    "https://www.accuweather.com/rss",  # AccuWeather
+    "https://www.weatherunderground.com/rss",  # Weather Underground
+    "https://www.noaa.gov/rss",  # NOAA News & Weather
+    "https://www.wunderground.com/weather/rss/",  # Weather Underground Forecasts
+    "https://www.climate.gov/rss",  # NOAA Climate.gov
+    "https://www.ecmwf.int/rss",  # European Centre for Medium-Range Weather Forecasts
+    "https://www.bom.gov.au/rss/feeds.xml",  # Australian Bureau of Meteorology
+
+    # Data & Statistics
+    "https://data.worldbank.org/rss",  # World Bank Data
+    "https://www.imf.org/external/np/weblogs/rss.aspx",  # IMF Blog & Data
+    "https://www.un.org/en/rss-feed",  # United Nations News & Data
+    "https://www.oecd.org/rss/",  # OECD Data & Publications
+    "https://www.statista.com/rss/",  # Statista Data Insights
+    "https://www.kff.org/rss/",  # Kaiser Family Foundation Health Data
+    "https://www.who.int/rss.xml",  # World Health Organization News & Data
+    "https://www.cdc.gov/rss/",  # CDC Data & Alerts
+    "https://www.bea.gov/rss/",  # U.S. Bureau of Economic Analysis
+    "https://www.census.gov/rss/",  # U.S. Census Bureau
+    "https://www.fdic.gov/rss/",  # FDIC Data & News
+
+    # Existing Categories (Expanded)
+    "https://www.nasa.gov/rss/dyn/breaking_news.rss",  # Space
+    "https://www.sciencedaily.com/rss/top/science.xml",  # Science
+    "https://www.wired.com/feed/rss",  # Tech
+    "https://www.horoscope.com/us/horoscopes/general/rss/horoscope-rss.aspx",  # Astrology
+    "http://rss.cnn.com/rss/cnn_allpolitics.rss",  # Politics
+    "https://phys.org/rss-feed/physics-news/",  # Physics
+    "https://www.spaceweatherlive.com/en/news/rss",  # Solar & Space
+    "https://weather.com/feeds/rss",  # Earth Weather
+    "https://feeds.bbci.co.uk/news/politics/rss.xml",  # Politics
+    "https://www.reuters.com/arc/outboundfeeds/newsletter-politics/?outputType=xml",  # Politics
+    "https://www.politico.com/rss/politics.xml",  # Politics
+    "https://thehill.com/feed/",  # Politics
+    "https://www.aps.org/publications/apsnews/updates/rss.cfm",  # Physics
+    "https://www.quantamagazine.org/feed/",  # Science
+    "https://www.sciencedaily.com/rss/matter_energy/physics.xml",  # Physics
+    "https://physicsworld.com/feed/",  # Physics
+    "https://www.swpc.noaa.gov/rss.xml",  # Solar & Space
+    "https://feeds.bbci.co.uk/weather/feeds/rss/5day/world/",  # Earth Weather
+    "https://www.weather.gov/rss",  # Earth Weather
+    "https://www.foxweather.com/rss",  # Earth Weather
+    "https://techcrunch.com/feed/",  # Tech
+    "https://arstechnica.com/feed/",  # Tech
+    "https://gizmodo.com/rss",  # Tech
+    "https://www.theverge.com/rss/index.xml",  # Tech
+    "https://www.space.com/feeds/all",  # Space
+    "https://www.universetoday.com/feed/",  # Space
+    "https://skyandtelescope.org/feed/",  # Space
+    "https://www.esa.int/rss",  # Space
+    "https://www.smithsonianmag.com/rss/",  # Science
+    "https://www.popsci.com/rss.xml",  # Science
+    "https://www.discovermagazine.com/rss",  # Science
+    "https://www.atlasobscura.com/feeds/latest",  # Science
+    "https://www.nature.com/nature.rss",  # Science
+    "https://www.scientificamerican.com/rss/",  # Science
+    "https://www.newscientist.com/feed/home/",  # Science
+    "https://www.livescience.com/feeds/all",  # Science
+    "https://astrostyle.com/feed/",  # Astrology
+    "https://www.vogue.com/feed/rss",  # Lifestyle
+]
+
+HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "YOUR_HF_API_TOKEN")
+REPO_ID = "broadfield-dev/news-rag-db"
+
+# Initialize Hugging Face API
+login(token=HF_API_TOKEN)
+hf_api = HfApi()
+
+# Initialize embedding model and vector DB
+embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_model)
+
+def fetch_rss_feeds():
+    articles = []
+    seen_keys = set()
+    for feed_url in RSS_FEEDS:
+        try:
+            logger.info(f"Fetching {feed_url}")
+            feed = feedparser.parse(feed_url)
+            if feed.bozo:
+                logger.warning(f"Parse error for {feed_url}: {feed.bozo_exception}")
+                continue
+            for entry in feed.entries:
+                title = entry.get("title", "No Title").strip()
+                link = entry.get("link", "").strip()
+                description = entry.get("summary", entry.get("description", "No Description"))
+                published = entry.get("published", "Unknown Date").strip()
+                key = f"{title}|{link}|{published}"
+                if key not in seen_keys:
+                    seen_keys.add(key)
+                    image = (entry.get("media_content", [{}])[0].get("url") or
+                             entry.get("media_thumbnail", [{}])[0].get("url") or "svg")
+                    articles.append({
+                        "title": title,
+                        "link": link,
+                        "description": description,
+                        "published": published,
+                        "category": categorize_feed(feed_url),
+                        "image": image,
+                    })
+        except Exception as e:
+            logger.error(f"Error fetching {feed_url}: {e}")
+    logger.info(f"Total articles fetched: {len(articles)}")
+    return articles
+
+def categorize_feed(url):
+    if "nature" in url or "science.org" in url or "arxiv.org" in url or "plos.org" in url or "annualreviews.org" in url or "journals.uchicago.edu" in url or "jneurosci.org" in url or "cell.com" in url or "nejm.org" in url or "lancet.com" in url:
+        return "Academic Papers"
+    elif "reuters.com/business" in url or "bloomberg.com" in url or "ft.com" in url or "marketwatch.com" in url or "cnbc.com" in url or "foxbusiness.com" in url or "wsj.com" in url or "bworldonline.com" in url or "economist.com" in url or "forbes.com" in url:
+        return "Business"
+    elif "investing.com" in url or "cnbc.com/market" in url or "marketwatch.com/market" in url or "fool.co.uk" in url or "zacks.com" in url or "seekingalpha.com" in url or "barrons.com" in url or "yahoofinance.com" in url:
+        return "Stocks & Markets"
+    elif "whitehouse.gov" in url or "state.gov" in url or "commerce.gov" in url or "transportation.gov" in url or "ed.gov" in url or "dol.gov" in url or "justice.gov" in url or "federalreserve.gov" in url or "occ.gov" in url or "sec.gov" in url or "bls.gov" in url or "usda.gov" in url or "gao.gov" in url or "cbo.gov" in url or "fema.gov" in url or "defense.gov" in url or "hhs.gov" in url or "energy.gov" in url or "interior.gov" in url:
+        return "Federal Government"
+    elif "weather.gov" in url or "metoffice.gov.uk" in url or "accuweather.com" in url or "weatherunderground.com" in url or "noaa.gov" in url or "wunderground.com" in url or "climate.gov" in url or "ecmwf.int" in url or "bom.gov.au" in url:
+        return "Weather"
+    elif "data.worldbank.org" in url or "imf.org" in url or "un.org" in url or "oecd.org" in url or "statista.com" in url or "kff.org" in url or "who.int" in url or "cdc.gov" in url or "bea.gov" in url or "census.gov" in url or "fdic.gov" in url:
+        return "Data & Statistics"
+    elif "nasa" in url or "spaceweatherlive" in url or "space" in url or "universetoday" in url or "skyandtelescope" in url or "esa" in url:
+        return "Space"
+    elif "sciencedaily" in url or "quantamagazine" in url or "smithsonianmag" in url or "popsci" in url or "discovermagazine" in url or "scientificamerican" in url or "newscientist" in url or "livescience" in url or "atlasobscura" in url:
+        return "Science"
+    elif "wired" in url or "techcrunch" in url or "arstechnica" in url or "gizmodo" in url or "theverge" in url:
+        return "Tech"
+    elif "horoscope" in url or "astrostyle" in url:
+        return "Astrology"
+    elif "cnn_allpolitics" in url or "bbci.co.uk/news/politics" in url or "reuters.com/arc/outboundfeeds/newsletter-politics" in url or "politico.com/rss/politics" in url or "thehill" in url:
+        return "Politics"
+    elif "weather" in url or "swpc.noaa.gov" in url or "foxweather" in url:
+        return "Earth Weather"
+    elif "vogue" in url:
+        return "Lifestyle"
+    elif "phys.org" in url or "aps.org" in url or "physicsworld" in url:
+        return "Physics"
+    return "Uncategorized"
+
+def process_and_store_articles(articles):
+    documents = []
+    for article in articles:
+        try:
+            metadata = {
+                "title": article["title"],
+                "link": article["link"],
+                "original_description": article["description"],
+                "published": article["published"],
+                "category": article["category"],
+                "image": article["image"],
+            }
+            doc = Document(page_content=article["description"], metadata=metadata)
+            documents.append(doc)
+        except Exception as e:
+            logger.error(f"Error processing article {article['title']}: {e}")
+    
+    if documents:
+        try:
+            vector_db.add_documents(documents)
+            logger.info(f"Stored {len(documents)} articles in DB")
+        except Exception as e:
+            logger.error(f"Error storing articles: {e}")
+
+def download_from_hf_hub():
+    if os.path.exists(LOCAL_DB_DIR):
+        shutil.rmtree(LOCAL_DB_DIR)
+    try:
+        hf_api.create_repo(repo_id=REPO_ID, repo_type="dataset", exist_ok=True, token=HF_API_TOKEN)
+        logger.info(f"Downloading Chroma DB from {REPO_ID}...")
+        hf_api.download_repo(repo_id=REPO_ID, repo_type="dataset", local_dir=LOCAL_DB_DIR, token=HF_API_TOKEN)
+    except Exception as e:
+        logger.error(f"Error downloading from Hugging Face Hub: {e}")
+        raise
+
+def upload_to_hf_hub():
+    if os.path.exists(LOCAL_DB_DIR):
+        try:
+            hf_api.create_repo(repo_id=REPO_ID, repo_type="dataset", exist_ok=True, token=HF_API_TOKEN)
+            logger.info(f"Uploading Chroma DB to {REPO_ID}...")
+            for root, _, files in os.walk(LOCAL_DB_DIR):
+                for file in files:
+                    local_path = os.path.join(root, file)
+                    remote_path = os.path.relpath(local_path, LOCAL_DB_DIR)
+                    hf_api.upload_file(
+                        path_or_fileobj=local_path,
+                        path_in_repo=remote_path,
+                        repo_id=REPO_ID,
+                        repo_type="dataset",
+                        token=HF_API_TOKEN
+                    )
+            logger.info(f"Database uploaded to: {REPO_ID}")
+        except Exception as e:
+            logger.error(f"Error uploading to Hugging Face Hub: {e}")
+            raise
+
+if __name__ == "__main__":
+    articles = fetch_rss_feeds()
+    process_and_store_articles(articles)
+    upload_to_hf_hub()