app.py

import os
import feedparser
from flask import Flask, render_template, request
from huggingface_hub import HfApi, Repository
from langchain_huggingface import HuggingFaceInferenceClient
from langchain.vectorstores import Chroma
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.docstore.document import Document
import requests
import shutil

# Flask app setup
app = Flask(__name__)

# Hugging Face setup
HF_API_TOKEN = os.getenv("HF_API_TOKEN", "YOUR_HF_API_TOKEN")
HF_MODEL = "Qwen/Qwen-72B-Instruct"
REPO_ID = "broadfield-dev/news-rag-db"
LOCAL_DB_DIR = "chroma_db"
client = HuggingFaceInferenceClient(model=HF_MODEL, api_key=HF_API_TOKEN)

RSS_FEEDS = [
    "https://www.sciencedaily.com/rss/top/science.xml",
    "https://www.horoscope.com/us/horoscopes/general/rss/horoscope-rss.aspx",
    "http://rss.cnn.com/rss/cnn_allpolitics.rss",
    "https://phys.org/rss-feed/physics-news/",
    "https://www.spaceweatherlive.com/en/news/rss",
    "https://weather.com/feeds/rss",
    "https://www.wired.com/feed/rss",
    "https://www.nasa.gov/rss/dyn/breaking_news.rss",
    "https://www.nationalgeographic.com/feed/",
    "https://www.nature.com/nature.rss",
    "https://www.scientificamerican.com/rss/",
    "https://www.newscientist.com/feed/home/",
    "https://www.livescience.com/feeds/all",
    "https://www.hindustantimes.com/feed/horoscope/rss",
    "https://www.washingtonpost.com/wp-srv/style/horoscopes/rss.xml",
    "https://astrostyle.com/feed/",
    "https://www.vogue.com/feed/rss",
    "https://feeds.bbci.co.uk/news/politics/rss.xml",
    "https://www.reuters.com/arc/outboundfeeds/newsletter-politics/?outputType=xml",
    "https://www.politico.com/rss/politics.xml",
    "https://thehill.com/feed/",
    "https://www.aps.org/publications/apsnews/updates/rss.cfm",
    "https://www.quantamagazine.org/feed/",
    "https://www.sciencedaily.com/rss/matter_energy/physics.xml",
    "https://physicsworld.com/feed/",
    "https://www.swpc.noaa.gov/rss.xml",
    "https://www.nasa.gov/rss/dyn/solar_system.rss",
    "https://weather.com/science/space/rss",
    "https://www.space.com/feeds/space-weather",
    "https://www.accuweather.com/en/rss",
    "https://feeds.bbci.co.uk/weather/feeds/rss/5day/world/",
    "https://www.weather.gov/rss",
    "https://www.foxweather.com/rss",
    "https://techcrunch.com/feed/",
    "https://arstechnica.com/feed/",
    "https://gizmodo.com/rss",
    "https://www.theverge.com/rss/index.xml",
    "https://www.space.com/feeds/all",
    "https://www.universetoday.com/feed/",
    "https://skyandtelescope.org/feed/",
    "https://www.esa.int/rss",
    "https://www.smithsonianmag.com/rss/",
    "https://www.popsci.com/rss.xml",
    "https://www.discovermagazine.com/rss",
    "https://www.atlasobscura.com/feeds/latest"
]

# Embedding model
embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
vector_db = Chroma(persist_directory=LOCAL_DB_DIR, embedding_function=embedding_model)
hf_api = HfApi()

def fetch_rss_feeds():
    articles = []
    for feed_url in RSS_FEEDS:
        feed = feedparser.parse(feed_url)
        for entry in feed.entries[:5]:  # Limit to 5 per feed
            articles.append({
                "title": entry.get("title", "No Title"),
                "link": entry.get("link", ""),
                "description": entry.get("summary", entry.get("description", "No Description")),
                "published": entry.get("published", "Unknown Date"),
                "category": categorize_feed(feed_url),
            })
    return articles

def categorize_feed(url):
    """Simple categorization based on URL."""
    if "sciencedaily" in url or "phys.org" in url:
        return "Science & Physics"
    elif "horoscope" in url:
        return "Astrology"
    elif "politics" in url:
        return "Politics"
    elif "spaceweather" in url or "nasa" in url:
        return "Solar & Space"
    elif "weather" in url:
        return "Earth Weather"
    else:
        return "Cool Stuff"

def summarize_article(text):
    prompt = f"Summarize the following text concisely:\n\n{text}"
    response = client.generate(prompt, max_new_tokens=100, temperature=0.7)
    return response.generated_text.strip()

def categorize_article(text):
    prompt = f"Classify the sentiment as positive, negative, or neutral:\n\n{text}"
    response = client.generate(prompt, max_new_tokens=10, temperature=0.7)
    return response.generated_text.strip()

def process_and_store_articles(articles):
    documents = []
    for article in articles:
        summary = summarize_article(article["description"])
        sentiment = categorize_article(article["description"])
        doc = Document(
            page_content=summary,
            metadata={
                "title": article["title"],
                "link": article["link"],
                "original_description": article["description"],
                "published": article["published"],
                "category": article["category"],
                "sentiment": sentiment,
            }
        )
        documents.append(doc)
    vector_db.add_documents(documents)
    vector_db.persist()
    upload_to_hf_hub()

def upload_to_hf_hub():
    if os.path.exists(LOCAL_DB_DIR):
        try:
            hf_api.create_repo(repo_id=REPO_ID, repo_type="dataset", exist_ok=True)
        except Exception as e:
            print(f"Error creating repo: {e}")
        for root, _, files in os.walk(LOCAL_DB_DIR):
            for file in files:
                local_path = os.path.join(root, file)
                remote_path = os.path.relpath(local_path, LOCAL_DB_DIR)
                hf_api.upload_file(
                    path_or_fileobj=local_path,
                    path_in_repo=remote_path,
                    repo_id=REPO_ID,
                    repo_type="dataset",
                    token=HF_API_TOKEN
                )
        print(f"Database uploaded to: {REPO_ID}")

@app.route('/', methods=['GET', 'POST'])
def index():
    articles = fetch_rss_feeds()
    process_and_store_articles(articles)
    stored_docs = vector_db.similarity_search("news", k=len(articles))
    enriched_articles = [
        {
            "title": doc.metadata["title"],
            "link": doc.metadata["link"],
            "summary": doc.page_content,
            "category": doc.metadata["category"],
            "sentiment": doc.metadata["sentiment"],
            "published": doc.metadata["published"],
        }
        for doc in stored_docs
    ]

    if request.method == 'POST':
        query = request.form.get('search')
        if query:
            results = vector_db.similarity_search(query, k=10)
            enriched_articles = [
                {
                    "title": doc.metadata["title"],
                    "link": doc.metadata["link"],
                    "summary": doc.page_content,
                    "category": doc.metadata["category"],
                    "sentiment": doc.metadata["sentiment"],
                    "published": doc.metadata["published"],
                }
                for doc in results
            ]

    # Organize by category
    categorized_articles = {}
    for article in enriched_articles:
        cat = article["category"]
        if cat not in categorized_articles:
            categorized_articles[cat] = []
        categorized_articles[cat].append(article)

    return render_template("index.html", categorized_articles=categorized_articles)

# Updated HTML template
HTML_TEMPLATE = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>News Feed Hub</title>
    <style>
        body {
            font-family: 'Arial', sans-serif;
            margin: 0;
            padding: 20px;
            background-color: #f4f4f9;
            color: #333;
        }
        h1 {
            text-align: center;
            color: #2c3e50;
        }
        .search-container {
            text-align: center;
            margin: 20px 0;
        }
        .search-bar {
            width: 50%;
            padding: 12px;
            font-size: 16px;
            border: 2px solid #3498db;
            border-radius: 25px;
            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
            outline: none;
            transition: border-color 0.3s;
        }
        .search-bar:focus {
            border-color: #2980b9;
        }
        .category-section {
            margin: 30px 0;
        }
        .category-title {
            background-color: #3498db;
            color: white;
            padding: 10px;
            border-radius: 5px;
            font-size: 1.4em;
        }
        .article {
            background-color: white;
            padding: 15px;
            margin: 10px 0;
            border-radius: 8px;
            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
            transition: transform 0.2s;
        }
        .article:hover {
            transform: translateY(-3px);
        }
        .title a {
            font-size: 1.2em;
            color: #2c3e50;
            text-decoration: none;
        }
        .title a:hover {
            color: #3498db;
        }
        .summary {
            color: #555;
            margin: 5px 0;
        }
        .sentiment {
            font-style: italic;
            color: #7f8c8d;
        }
        .published {
            font-size: 0.9em;
            color: #95a5a6;
        }
    </style>
</head>
<body>
    <h1>News Feed Hub</h1>
    <div class="search-container">
        <form method="POST">
            <input type="text" name="search" class="search-bar" placeholder="Search news semantically...">
        </form>
    </div>
    {% for category, articles in categorized_articles.items() %}
    <div class="category-section">
        <div class="category-title">{{ category }}</div>
        {% for article in articles %}
        <div class="article">
            <div class="title"><a href="{{ article.link }}" target="_blank">{{ article.title }}</a></div>
            <div class="summary">{{ article.summary }}</div>
            <div class="sentiment">Sentiment: {{ article.sentiment }}</div>
            <div class="published">Published: {{ article.published }}</div>
        </div>
        {% endfor %}
    </div>
    {% endfor %}
</body>
</html>
"""

if __name__ == "__main__":
    os.makedirs("templates", exist_ok=True)
    with open("templates/index.html", "w") as f:
        f.write(HTML_TEMPLATE)
    if os.path.exists(LOCAL_DB_DIR):
        shutil.rmtree(LOCAL_DB_DIR)
    app.run(host="0.0.0.0", port=7560)