Spaces:
Sleeping
Sleeping
Update rss_processor.py
Browse files- rss_processor.py +2 -128
rss_processor.py
CHANGED
|
@@ -6,140 +6,14 @@ from langchain.docstore.document import Document
|
|
| 6 |
import logging
|
| 7 |
from huggingface_hub import HfApi, login
|
| 8 |
import shutil
|
| 9 |
-
|
| 10 |
# Setup logging
|
| 11 |
logging.basicConfig(level=logging.INFO)
|
| 12 |
logger = logging.getLogger(__name__)
|
| 13 |
|
| 14 |
# Constants
|
| 15 |
LOCAL_DB_DIR = "chroma_db"
|
| 16 |
-
RSS_FEEDS =
|
| 17 |
-
# Academic Papers (Published Papers Across Fields)
|
| 18 |
-
"https://www.nature.com/nature/current_issue/rss", # Nature - High-brow science
|
| 19 |
-
"https://www.science.org/action/showFeed?type=etoc&feed=rss", # Science journal
|
| 20 |
-
"https://arxiv.org/rss/cs", # Computer Science arXiv
|
| 21 |
-
"https://arxiv.org/rss/physics", # Physics arXiv
|
| 22 |
-
"https://arxiv.org/rss/math", # Mathematics arXiv
|
| 23 |
-
"https://arxiv.org/rss/astro-ph", # Astrophysics arXiv
|
| 24 |
-
"https://arxiv.org/rss/biology", # Biology arXiv
|
| 25 |
-
"https://arxiv.org/rss/econ", # Economics arXiv
|
| 26 |
-
"https://arxiv.org/rss/stat", # Statistics arXiv
|
| 27 |
-
"https://arxiv.org/rss/quant-ph", # Quantum Physics arXiv
|
| 28 |
-
"https://www.plos.org/feed/", # PLOS (Public Library of Science) - multidisciplinary
|
| 29 |
-
"https://www.journals.uchicago.edu/action/showFeed?type=etoc&feed=rss&jc=pnl", # Philosophy (e.g., Philosophy of Science)
|
| 30 |
-
"https://www.annualreviews.org/action/showFeed?type=etoc&feed=rss&jc=anrevo", # Annual Reviews (various fields)
|
| 31 |
-
"https://www.jneurosci.org/rss/current.xml", # Journal of Neuroscience
|
| 32 |
-
"https://www.cell.com/cell/rss", # Cell Press - Biology/Medicine
|
| 33 |
-
"https://www.nejm.org/rss/all-articles.xml", # New England Journal of Medicine
|
| 34 |
-
"https://www.lancet.com/rss/lancet_current.xml", # The Lancet - Medicine
|
| 35 |
-
|
| 36 |
-
# Business
|
| 37 |
-
"https://www.reuters.com/arc/outboundfeeds/business-news/?outputType=rss", # Reuters Business
|
| 38 |
-
"https://www.bloomberg.com/feeds/rss/businessweek.xml", # Bloomberg Businessweek
|
| 39 |
-
"https://www.ft.com/rss/home", # Financial Times
|
| 40 |
-
"https://www.marketwatch.com/rss/topstories", # MarketWatch Business & Finance
|
| 41 |
-
"https://www.cnbc.com/id/100003114/device/rss/rss.html", # CNBC Business News
|
| 42 |
-
"https://www.foxbusiness.com/rss", # Fox Business
|
| 43 |
-
"https://www.wsj.com/xml/rss/3_7085.xml", # Wall Street Journal - Business & Finance
|
| 44 |
-
"https://www.bworldonline.com/feed/", # BusinessWorld Online (Philippines)
|
| 45 |
-
"https://www.economist.com/business/rss.xml", # The Economist - Business
|
| 46 |
-
"https://www.forbes.com/business/feed/", # Forbes Business
|
| 47 |
-
|
| 48 |
-
# Stocks & Markets
|
| 49 |
-
"https://www.investing.com/rss/news_25.rss", # Investing.com Stocks & Markets
|
| 50 |
-
"https://www.cnbc.com/id/100727362/device/rss/rss.html", # CNBC Market Data
|
| 51 |
-
"https://www.marketwatch.com/rss/marketpulse", # MarketWatch Market News
|
| 52 |
-
"https://www.fool.co.uk/feed/", # Motley Fool UK - Stock Investing
|
| 53 |
-
"https://www.zacks.com/rss/zc.xml", # Zacks Investment Research
|
| 54 |
-
"https://seekingalpha.com/feed.xml", # Seeking Alpha - Stock Analysis
|
| 55 |
-
"https://www.barrons.com/rss.xml", # Barron's - Market News
|
| 56 |
-
"https://www.yahoofinance.com/news/rss", # Yahoo Finance - Stocks & Markets
|
| 57 |
-
|
| 58 |
-
# Federal Government
|
| 59 |
-
"https://www.whitehouse.gov/feed/", # White House News
|
| 60 |
-
"https://www.state.gov/rss-feeds/", # U.S. Department of State
|
| 61 |
-
"https://www.commerce.gov/feeds/rss", # U.S. Department of Commerce
|
| 62 |
-
"https://www.transportation.gov/feeds/rss", # U.S. Department of Transportation
|
| 63 |
-
"https://www.ed.gov/rss/", # U.S. Department of Education
|
| 64 |
-
"https://www.dol.gov/newsroom/rss-feeds", # U.S. Department of Labor
|
| 65 |
-
"https://www.justice.gov/feeds/opa", # U.S. Department of Justice
|
| 66 |
-
"https://www.federalreserve.gov/feeds/", # Federal Reserve Board
|
| 67 |
-
"https://www.occ.gov/rss-feeds.html", # Office of the Comptroller of the Currency
|
| 68 |
-
"https://www.sec.gov/news/sec-rss-feeds.htm", # SEC News
|
| 69 |
-
"https://www.bls.gov/rss/", # U.S. Bureau of Labor Statistics
|
| 70 |
-
"https://www.usda.gov/rss/", # U.S. Department of Agriculture
|
| 71 |
-
"https://www.gao.gov/rss/", # Government Accountability Office
|
| 72 |
-
"https://www.cbo.gov/rss/", # Congressional Budget Office
|
| 73 |
-
"https://www.fema.gov/rss/", # Federal Emergency Management Agency
|
| 74 |
-
"https://www.defense.gov/Newsroom/RSS/", # U.S. Department of Defense
|
| 75 |
-
"https://www.hhs.gov/rss/", # U.S. Department of Health & Human Services
|
| 76 |
-
"https://www.energy.gov/rss/", # U.S. Department of Energy
|
| 77 |
-
"https://www.interior.gov/rss", # U.S. Department of the Interior
|
| 78 |
-
|
| 79 |
-
# Weather
|
| 80 |
-
"https://www.weather.gov/rss/", # National Weather Service
|
| 81 |
-
"https://www.metoffice.gov.uk/weather/warnings-and-advice/uk-warnings/rss", # UK Met Office
|
| 82 |
-
"https://www.accuweather.com/rss", # AccuWeather
|
| 83 |
-
"https://www.weatherunderground.com/rss", # Weather Underground
|
| 84 |
-
"https://www.noaa.gov/rss", # NOAA News & Weather
|
| 85 |
-
"https://www.wunderground.com/weather/rss/", # Weather Underground Forecasts
|
| 86 |
-
"https://www.climate.gov/rss", # NOAA Climate.gov
|
| 87 |
-
"https://www.ecmwf.int/rss", # European Centre for Medium-Range Weather Forecasts
|
| 88 |
-
"https://www.bom.gov.au/rss/feeds.xml", # Australian Bureau of Meteorology
|
| 89 |
-
|
| 90 |
-
# Data & Statistics
|
| 91 |
-
"https://data.worldbank.org/rss", # World Bank Data
|
| 92 |
-
"https://www.imf.org/external/np/weblogs/rss.aspx", # IMF Blog & Data
|
| 93 |
-
"https://www.un.org/en/rss-feed", # United Nations News & Data
|
| 94 |
-
"https://www.oecd.org/rss/", # OECD Data & Publications
|
| 95 |
-
"https://www.statista.com/rss/", # Statista Data Insights
|
| 96 |
-
"https://www.kff.org/rss/", # Kaiser Family Foundation Health Data
|
| 97 |
-
"https://www.who.int/rss.xml", # World Health Organization News & Data
|
| 98 |
-
"https://www.cdc.gov/rss/", # CDC Data & Alerts
|
| 99 |
-
"https://www.bea.gov/rss/", # U.S. Bureau of Economic Analysis
|
| 100 |
-
"https://www.census.gov/rss/", # U.S. Census Bureau
|
| 101 |
-
"https://www.fdic.gov/rss/", # FDIC Data & News
|
| 102 |
-
|
| 103 |
-
# Existing Categories (Expanded)
|
| 104 |
-
"https://www.nasa.gov/rss/dyn/breaking_news.rss", # Space
|
| 105 |
-
"https://www.sciencedaily.com/rss/top/science.xml", # Science
|
| 106 |
-
"https://www.wired.com/feed/rss", # Tech
|
| 107 |
-
"https://www.horoscope.com/us/horoscopes/general/rss/horoscope-rss.aspx", # Astrology
|
| 108 |
-
"http://rss.cnn.com/rss/cnn_allpolitics.rss", # Politics
|
| 109 |
-
"https://phys.org/rss-feed/physics-news/", # Physics
|
| 110 |
-
"https://www.spaceweatherlive.com/en/news/rss", # Solar & Space
|
| 111 |
-
"https://weather.com/feeds/rss", # Earth Weather
|
| 112 |
-
"https://feeds.bbci.co.uk/news/politics/rss.xml", # Politics
|
| 113 |
-
"https://www.reuters.com/arc/outboundfeeds/newsletter-politics/?outputType=xml", # Politics
|
| 114 |
-
"https://www.politico.com/rss/politics.xml", # Politics
|
| 115 |
-
"https://thehill.com/feed/", # Politics
|
| 116 |
-
"https://www.aps.org/publications/apsnews/updates/rss.cfm", # Physics
|
| 117 |
-
"https://www.quantamagazine.org/feed/", # Science
|
| 118 |
-
"https://www.sciencedaily.com/rss/matter_energy/physics.xml", # Physics
|
| 119 |
-
"https://physicsworld.com/feed/", # Physics
|
| 120 |
-
"https://www.swpc.noaa.gov/rss.xml", # Solar & Space
|
| 121 |
-
"https://feeds.bbci.co.uk/weather/feeds/rss/5day/world/", # Earth Weather
|
| 122 |
-
"https://www.weather.gov/rss", # Earth Weather
|
| 123 |
-
"https://www.foxweather.com/rss", # Earth Weather
|
| 124 |
-
"https://techcrunch.com/feed/", # Tech
|
| 125 |
-
"https://arstechnica.com/feed/", # Tech
|
| 126 |
-
"https://gizmodo.com/rss", # Tech
|
| 127 |
-
"https://www.theverge.com/rss/index.xml", # Tech
|
| 128 |
-
"https://www.space.com/feeds/all", # Space
|
| 129 |
-
"https://www.universetoday.com/feed/", # Space
|
| 130 |
-
"https://skyandtelescope.org/feed/", # Space
|
| 131 |
-
"https://www.esa.int/rss", # Space
|
| 132 |
-
"https://www.smithsonianmag.com/rss/", # Science
|
| 133 |
-
"https://www.popsci.com/rss.xml", # Science
|
| 134 |
-
"https://www.discovermagazine.com/rss", # Science
|
| 135 |
-
"https://www.atlasobscura.com/feeds/latest", # Science
|
| 136 |
-
"https://www.nature.com/nature.rss", # Science
|
| 137 |
-
"https://www.scientificamerican.com/rss/", # Science
|
| 138 |
-
"https://www.newscientist.com/feed/home/", # Science
|
| 139 |
-
"https://www.livescience.com/feeds/all", # Science
|
| 140 |
-
"https://astrostyle.com/feed/", # Astrology
|
| 141 |
-
"https://www.vogue.com/feed/rss", # Lifestyle
|
| 142 |
-
]
|
| 143 |
|
| 144 |
HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "YOUR_HF_API_TOKEN")
|
| 145 |
REPO_ID = "broadfield-dev/news-rag-db"
|
|
|
|
| 6 |
import logging
|
| 7 |
from huggingface_hub import HfApi, login
|
| 8 |
import shutil
|
| 9 |
+
import rss_feeds
|
| 10 |
# Setup logging
|
| 11 |
logging.basicConfig(level=logging.INFO)
|
| 12 |
logger = logging.getLogger(__name__)
|
| 13 |
|
| 14 |
# Constants
|
| 15 |
LOCAL_DB_DIR = "chroma_db"
|
| 16 |
+
RSS_FEEDS = rss_feeds.RSS_FEEDS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "YOUR_HF_API_TOKEN")
|
| 19 |
REPO_ID = "broadfield-dev/news-rag-db"
|