Update rss_processor.py

rss_processor.py

@@ -16,7 +16,7 @@ HF_API_TOKEN = os.getenv("DEMO_HF_API_TOKEN", "YOUR_HF_API_TOKEN")
 REPO_ID = "broadfield-dev/news-rag-db"
 LOCAL_DB_DIR = "chroma_db"
 
-# Explicitly login to Hugging Face Hub (no InferenceClient needed anymore)
+# Explicitly login to Hugging Face Hub
 login(token=HF_API_TOKEN)
 hf_api = HfApi()
 
@@ -77,14 +77,14 @@ def fetch_rss_feeds():
                 logger.warning(f"Failed to parse {feed_url}: {feed.bozo_exception}")
                 continue
             for entry in feed.entries[:5]:
-                image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or None
+                image = entry.get("media_content", [{}])[0].get("url") or entry.get("media_thumbnail", [{}])[0].get("url") or ""
                 articles.append({
                     "title": entry.get("title", "No Title"),
                     "link": entry.get("link", ""),
                     "description": entry.get("summary", entry.get("description", "No Description")),
                     "published": entry.get("published", "Unknown Date"),
                     "category": categorize_feed(feed_url),
-                    "image": image,
+                    "image": image if image else "",
                 })
             logger.info(f"Processed {len(feed.entries[:5])} entries from {feed_url}")
         except Exception as e:
@@ -109,23 +109,28 @@ def process_and_store_articles(articles):
     documents = []
     for article in articles:
         try:
+            # Ensure no None values in metadata
+            metadata = {
+                "title": article["title"] or "No Title",
+                "link": article["link"] or "",
+                "original_description": article["description"] or "No Description",
+                "published": article["published"] or "Unknown Date",
+                "category": article["category"] or "Uncategorized",
+                "image": article["image"] or "",
+            }
             doc = Document(
-                page_content=article["description"],
-                metadata={
-                    "title": article["title"],
-                    "link": article["link"],
-                    "original_description": article["description"],
-                    "published": article["published"],
-                    "category": article["category"],
-                    "image": article["image"],
-                }
+                page_content=article["description"] or "No Description",
+                metadata=metadata
             )
             documents.append(doc)
         except Exception as e:
             logger.error(f"Error processing article {article['title']}: {e}")
-    vector_db.add_documents(documents)
-    vector_db.persist()
-    logger.info("Vector DB persisted")
+    try:
+        vector_db.add_documents(documents)
+        vector_db.persist()
+        logger.info("Vector DB persisted")
+    except Exception as e:
+        logger.error(f"Error adding documents to vector DB: {e}")
     upload_to_hf_hub()
 
 def upload_to_hf_hub():