Update app.py

app.py

@@ -6,13 +6,19 @@ import hashlib
 import threading
 from pathlib import Path
 import logging
+from typing import List, Tuple
 
 import gradio as gr
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
-from selenium.common.exceptions import WebDriverException, NoSuchElementException
+from selenium.common.exceptions import (
+    WebDriverException,
+    NoSuchElementException,
+    TimeoutException,
+    StaleElementReferenceException,
+)
 from webdriver_manager.chrome import ChromeDriverManager
 from huggingface_hub import InferenceClient
 import mysql.connector
@@ -20,7 +26,7 @@ import feedparser  # For parsing RSS feeds
 import sqlite3  # For simple local storage if needed
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(filename)s - %(lineno)d - %(message)s')
 
 # Configuration (replace with your actual values or environment variables)
 DB_HOST = os.environ.get("DB_HOST", "your_host")
@@ -28,6 +34,9 @@ DB_USER = os.environ.get("DB_USER", "your_user")
 DB_PASSWORD = os.environ.get("DB_PASSWORD", "your_password")
 DB_NAME = os.environ.get("DB_NAME", "your_database")
 HUGGINGFACE_API_KEY = os.environ.get("HUGGINGFACE_API_KEY")  # Add API key
+DEFAULT_MONITORING_INTERVAL = 300  # 5 minutes in seconds
+MAX_MONITORING_INTERVAL = 600  # 10 minutes in seconds
+CHANGE_FREQUENCY_THRESHOLD = 3  # Number of changes to trigger faster monitoring
 
 # Global variables
 monitoring_thread = None
@@ -36,6 +45,7 @@ db_connection = None
 current_task = None
 history = []
 url_monitoring_intervals = {}  # Store monitoring intervals for each URL
+change_counts = {}  # Track change frequency for each URL
 
 # Function to establish a database connection
 def get_db_connection():
@@ -77,28 +87,36 @@ def create_articles_table():
 create_articles_table()
 
 # Function to monitor URLs for changes
-def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
-    global history, url_monitoring_intervals
+def monitor_urls(target_urls: List[str], storage_location: str, feed_rss: bool, stop_event: threading.Event):
+    global history, url_monitoring_intervals, change_counts
     previous_hashes = {url: "" for url in target_urls}
     options = Options()
     options.headless = True
     options.add_argument("--disable-gpu")
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-dev-shm-usage")
+    options.add_experimental_option("excludeSwitches", ["enable-logging"])  # Suppress unnecessary logs
 
     driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
+    driver.implicitly_wait(10)  # Implicit wait for elements
 
     try:
         while not stop_event.is_set():
             for url in target_urls:
                 try:
                     # Dynamic monitoring interval
-                    interval = url_monitoring_intervals.get(url, 300)  # Default 5 minutes
+                    interval = url_monitoring_intervals.get(url, DEFAULT_MONITORING_INTERVAL)
 
                     driver.get(url)
                     time.sleep(2)  # Allow page to load
-                    current_content = driver.page_source
-                    current_hash = hashlib.md5(current_content.encode('utf-8')).hexdigest()
+
+                    # Check for changes
+                    try:
+                        current_content = driver.find_element(By.TAG_NAME, "body").get_attribute("innerHTML")
+                        current_hash = hashlib.md5(current_content.encode('utf-8')).hexdigest()
+                    except (NoSuchElementException, TimeoutException, StaleElementReferenceException) as e:
+                        logging.warning(f"Error getting content for {url}: {e}")
+                        continue
 
                     if current_hash != previous_hashes[url]:
                         previous_hashes[url] = current_hash
@@ -118,12 +136,17 @@ def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
                         if feed_rss:
                             save_to_database(url, title, current_content, current_hash)
 
-                        # Adjust monitoring interval based on change frequency (example)
-                        url_monitoring_intervals[url] = 60  # Check more frequently after a change
+                        # Adjust monitoring interval based on change frequency
+                        change_counts[url] = change_counts.get(url, 0) + 1
+                        if change_counts[url] >= CHANGE_FREQUENCY_THRESHOLD:
+                            url_monitoring_intervals[url] = 60  # Check more frequently after multiple changes
+                        else:
+                            url_monitoring_intervals[url] = min(interval + 60, MAX_MONITORING_INTERVAL)  # Gradually increase interval
 
                     else:
-                        # Increase interval if no changes detected (example)
-                        url_monitoring_intervals[url] = min(url_monitoring_intervals[url] + 60, 600) # Max 10 min
+                        # Increase interval if no changes detected
+                        change_counts[url] = 0  # Reset change count if no change
+                        url_monitoring_intervals[url] = min(interval + 60, MAX_MONITORING_INTERVAL)
 
                 except WebDriverException as e:
                     logging.error(f"Error accessing {url}: {e}")
@@ -132,7 +155,7 @@ def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
                     break  # Exit inner loop if stop event is set
 
             if not stop_event.is_set():
-                time.sleep(interval) 
+                time.sleep(interval)
 
     except Exception as e:
         logging.error(f"Unexpected error in monitoring thread: {e}")
@@ -141,7 +164,7 @@ def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
         logging.info("Monitoring thread has been stopped.")
 
 # Function to save data to local storage (CSV)
-def save_to_storage(storage_location, url, title, content, timestamp):
+def save_to_storage(storage_location: str, url: str, title: str, content: str, timestamp: datetime.datetime):
     try:
         with open(storage_location, "a", newline='', encoding='utf-8') as csvfile:
             csv_writer = csv.writer(csvfile)
@@ -150,7 +173,7 @@ def save_to_storage(storage_location, url, title, content, timestamp):
         logging.error(f"Error saving to storage: {e}")
 
 # Function to save data to the database
-def save_to_database(url, title, content, hash):
+def save_to_database(url: str, title: str, content: str, hash: str):
     conn = get_db_connection()
     if conn:
         cursor = conn.cursor()
@@ -195,14 +218,15 @@ def generate_rss_feed():
     return None
 
 # Function to start monitoring
-def start_monitoring(target_urls, storage_location, feed_rss):
-    global monitoring_thread, stop_event, current_task, history
+def start_monitoring(target_urls: List[str], storage_location: str, feed_rss: bool):
+    global monitoring_thread, stop_event, current_task, history, change_counts
     if monitoring_thread and monitoring_thread.is_alive():
         return "Monitoring is already running.", history
 
     stop_event.clear()
     current_task = f"Monitoring URLs: {', '.join(target_urls)}"
     history.append(f"Task started: {current_task}")
+    change_counts = {url: 0 for url in target_urls}  # Reset change counts
     monitoring_thread = threading.Thread(
         target=monitor_urls,
         args=(target_urls, storage_location, feed_rss, stop_event),
@@ -224,7 +248,7 @@ def stop_monitoring():
         return "No monitoring task is currently running.", history
 
 # Function to handle chatbot responses
-def chatbot_response(message, history):
+def chatbot_response(message: str, history: List[Tuple[str, str]]):
     try:
         client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1", token=HUGGINGFACE_API_KEY)
         response = client.inference(message)
@@ -279,15 +303,15 @@ with gr.Blocks() as demo:
     # --- Event Handlers ---
 
     # Start monitoring button click
-    def on_start_click(target_urls_str, storage_loc, feed_enabled):
+    def on_start_click(target_urls_str: str, storage_loc: str, feed_enabled: bool):
         global history, url_monitoring_intervals
         try:
-            target_urls = [url.strip() for url in target_urls_str.split(",")]
+            target_urls = [url.strip() for url.strip() in target_urls_str.split(",")]
             if not all(target_urls):
                 return "Please enter valid URLs.", history
-            
+
             # Reset monitoring intervals when starting
-            url_monitoring_intervals = {url: 300 for url in target_urls} 
+            url_monitoring_intervals = {url: DEFAULT_MONITORING_INTERVAL for url in target_urls}
 
             status, history = start_monitoring(target_urls, storage_loc if storage_loc else None, feed_enabled)
             return status, history