Update app.py

app.py

@@ -1,276 +1,842 @@
-import mysql.connector
-from mysql.connector import errorcode
 import os
-import logging
 import time
 import hashlib
+import logging
 import datetime
-import gradio as gr
 import csv
+import threading
+import re
 from urllib.parse import urlparse
+
+import pandas as pd
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.keys import Keys
-from selenium.common.exceptions import NoSuchElementException
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
-from transformers import pipeline
-import feedparser
-from bs4 import BeautifulSoup
-import threading
+from selenium.common.exceptions import (
+    TimeoutException,
+    NoSuchElementException,
+    StaleElementReferenceException,
+)
+from webdriver_manager.chrome import ChromeDriverManager
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+import gradio as gr
+import xml.etree.ElementTree as ET
+import torch
+import mysql.connector
+from mysql.connector import errorcode
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
 
 # Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
 
 # Define constants
 DEFAULT_FILE_PATH = "scraped_data"
-PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
+PURPOSE = (
+    "You go to Culvers sites, you continuously seek changes on them since your last observation. "
+    "Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
+)
+
+# Global variables for task management
 HISTORY = []
 CURRENT_TASK = None
-STOP_THREADS = False
-
-# Define database configuration
-db_config = {
-    'user': os.getenv('DB_USER'),
-    'password': os.getenv('DB_PASSWORD'),
-    'host': os.getenv('DB_HOST'),
-    'raise_on_warnings': True
-}
-
-# Define a function to initialize the database
-def initialize_database(config):
+STOP_THREADS = False  # Flag to stop scraping threads
+
+# MySQL Database Connection
+def get_db_connection():
+    """
+    Establishes and returns a MySQL database connection using environment variables.
+    """
     try:
-        cnx = mysql.connector.connect(**config)
-        cursor = cnx.cursor()
-        
-        # Create database if it doesn't exist
-        cursor.execute("CREATE DATABASE IF NOT EXISTS scraper_db")
-        cnx.database = 'scraper_db'
-        
-        # Create tables
-        TABLES = {}
-        TABLES['scraped_data'] = (
-            "CREATE TABLE IF NOT EXISTS scraped_data ("
-            "  id INT AUTO_INCREMENT PRIMARY KEY,"
-            "  url VARCHAR(255) NOT NULL,"
-            "  content_hash VARCHAR(64) NOT NULL,"
-            "  change_detected DATETIME NOT NULL"
-            ") ENGINE=InnoDB"
+        connection = mysql.connector.connect(
+            host=os.getenv("DB_HOST"),
+            user=os.getenv("DB_USER"),
+            password=os.getenv("DB_PASSWORD"),
+            database=os.getenv("DB_NAME")
         )
-        
-        for table_name in TABLES:
-            table_description = TABLES[table_name]
-            try:
-                cursor.execute(table_description)
-                logging.info(f"Table `{table_name}` created successfully.")
-            except mysql.connector.Error as err:
-                if err.errno == errorcode.ER_TABLE_EXISTS_ERROR:
-                    logging.warning(f"Table `{table_name}` already exists.")
-                else:
-                    logging.error(err.msg)
-        
+        if connection.is_connected():
+            logging.info("Connected to MySQL database.")
+            return connection
+    except mysql.connector.Error as err:
+        if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
+            logging.error("Invalid database credentials.")
+        elif err.errno == errorcode.ER_BAD_DB_ERROR:
+            logging.error("Database does not exist.")
+        else:
+            logging.error(err)
+    return None
+
+# Initialize Database
+def initialize_database():
+    """
+    Initializes the database by creating necessary tables if they do not exist.
+    """
+    connection = get_db_connection()
+    if connection is None:
+        logging.error("Failed to connect to the database. Initialization aborted.")
+        return
+
+    cursor = connection.cursor()
+    try:
+        # Create table for scraped data
+        create_scraped_data_table = """
+        CREATE TABLE IF NOT EXISTS scraped_data (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            url VARCHAR(255) NOT NULL,
+            content_hash VARCHAR(64) NOT NULL,
+            change_detected DATETIME NOT NULL
+        )
+        """
+        cursor.execute(create_scraped_data_table)
+        logging.info("Table 'scraped_data' is ready.")
+
+        # Create table for action logs
+        create_action_logs_table = """
+        CREATE TABLE IF NOT EXISTS action_logs (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            action VARCHAR(255) NOT NULL,
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        cursor.execute(create_action_logs_table)
+        logging.info("Table 'action_logs' is ready.")
+
+    except mysql.connector.Error as err:
+        logging.error(f"Error creating tables: {err}")
+    finally:
         cursor.close()
-        cnx.close()
+        connection.close()
         logging.info("Database initialization complete.")
-    except mysql.connector.Error as err:
-        logging.error(f"Database initialization failed: {err}")
 
-# Define a function to start scraping
-def start_scraping(storage_location, urls, scrape_interval, content_type, db_config):
+# Function to monitor URLs for changes
+def monitor_urls(
+    storage_location: str,
+    urls: list,
+    scrape_interval: int,
+    content_type: str,
+    selector: str = None,
+):
+    """
+    Monitors the specified URLs for changes and logs any detected changes to the database.
+    """
+    global HISTORY, STOP_THREADS
+    previous_hashes = {url: "" for url in urls}
+
+    options = Options()
+    options.add_argument("--headless")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+
+    driver = create_driver(options)
+    if driver is None:
+        logging.error("WebDriver could not be initialized. Exiting monitor.")
+        return
+
+    try:
+        while not STOP_THREADS:
+            for url in urls:
+                try:
+                    driver.get(url)
+                    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+                    time.sleep(2)  # Additional wait for dynamic content
+
+                    if content_type == "text":
+                        current_content = driver.page_source
+                    elif content_type == "media":
+                        if selector:
+                            try:
+                                elements = WebDriverWait(driver, 5).until(
+                                    EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))
+                                )
+                                current_content = [element.get_attribute("src") for element in elements]
+                            except TimeoutException:
+                                logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")
+                                current_content = []
+                        else:
+                            elements = driver.find_elements(By.TAG_NAME, "img")
+                            current_content = [element.get_attribute("src") for element in elements]
+                    else:
+                        current_content = driver.page_source
+
+                    current_hash = hashlib.md5(str(current_content).encode("utf-8")).hexdigest()
+                    if current_hash != previous_hashes[url]:
+                        previous_hashes[url] = current_hash
+                        date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                        HISTORY.append(f"Change detected at {url} on {date_time_str}")
+
+                        # Insert change into MySQL database
+                        connection = get_db_connection()
+                        if connection:
+                            cursor = connection.cursor()
+                            insert_query = """
+                            INSERT INTO scraped_data (url, content_hash, change_detected)
+                            VALUES (%s, %s, %s)
+                            """
+                            cursor.execute(insert_query, (url, current_hash, date_time_str))
+                            connection.commit()
+                            cursor.close()
+                            connection.close()
+                            logging.info(f"Change detected at {url} on {date_time_str} and logged to database.")
+                        else:
+                            logging.error("Failed to connect to database. Change not logged.")
+
+                except (
+                    NoSuchElementException,
+                    StaleElementReferenceException,
+                    TimeoutException,
+                    Exception,
+                ) as e:
+                    logging.error(f"Error accessing {url}: {e}")
+            time.sleep(scrape_interval * 60)  # Wait for the next scrape interval
+    finally:
+        driver.quit()
+        logging.info("ChromeDriver session ended.")
+
+# Function to create WebDriver
+def create_driver(options: Options) -> webdriver.Chrome:
+    """
+    Initializes and returns a Selenium Chrome WebDriver instance.
+    """
+    try:
+        driver = webdriver.Chrome(
+            service=Service(ChromeDriverManager().install()), options=options
+        )
+        logging.info("ChromeDriver initialized successfully.")
+        return driver
+    except Exception as exception:
+        logging.error(f"Error initializing ChromeDriver: {exception}")
+        return None
+
+# Function to get initial observation
+def get_initial_observation(
+    driver: webdriver.Chrome, url: str, content_type: str, selector: str = None
+) -> str:
+    """
+    Retrieves the initial content from the URL and returns its MD5 hash.
+    """
+    try:
+        driver.get(url)
+        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+        time.sleep(2)  # Additional wait for dynamic content
+
+        if content_type == "text":
+            initial_content = driver.page_source
+        elif content_type == "media":
+            if selector:
+                try:
+                    elements = WebDriverWait(driver, 5).until(
+                        EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))
+                    )
+                    initial_content = [element.get_attribute("src") for element in elements]
+                except TimeoutException:
+                    logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")
+                    initial_content = []
+            else:
+                elements = driver.find_elements(By.TAG_NAME, "img")
+                initial_content = [element.get_attribute("src") for element in elements]
+        else:
+            initial_content = driver.page_source
+
+        initial_hash = hashlib.md5(str(initial_content).encode("utf-8")).hexdigest()
+        logging.info(f"Initial hash for {url}: {initial_hash}")
+        return initial_hash
+    except Exception as exception:
+        logging.error(f"Error accessing {url}: {exception}")
+        return None
+
+# Function to start scraping
+def start_scraping(
+    storage_location: str,
+    urls: str,
+    scrape_interval: int,
+    content_type: str,
+    selector: str = None,
+) -> str:
+    """
+    Starts the scraping process in a separate thread.
+    """
     global CURRENT_TASK, HISTORY, STOP_THREADS
-    
-    CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
+
+    if STOP_THREADS:
+        STOP_THREADS = False  # Reset the flag if previously stopped
+
+    url_list = [url.strip() for url in urls.split(",") if url.strip()]
+    CURRENT_TASK = f"Monitoring URLs: {', '.join(url_list)}"
     HISTORY.append(f"Task started: {CURRENT_TASK}")
-    
-    for url in urls:
-        # Create a folder for the URL
+    logging.info(f"Task started: {CURRENT_TASK}")
+
+    # Initialize database tables
+    initialize_database()
+
+    for url in url_list:
+        # Create a folder for the URL (if still needed for CSVs)
         hostname = urlparse(url).hostname
         folder_path = os.path.join(storage_location, hostname)
         os.makedirs(folder_path, exist_ok=True)
-        
+
         # Log the initial observation
         try:
-            with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
-                driver.get(url)
-                time.sleep(2)  # Wait for the page to load
-                if content_type == "text":
-                    initial_content = driver.page_source
-                elif content_type == "media":
-                    initial_content = driver.find_elements(By.TAG_NAME, "img")
-                else:
-                    initial_content = driver.page_source
-                initial_hash = hashlib.md5(str(initial_content).encode('utf-8')).hexdigest()
+            options = Options()
+            options.add_argument("--headless")
+            options.add_argument("--no-sandbox")
+            options.add_argument("--disable-dev-shm-usage")
+
+            driver = create_driver(options)
+            if driver is None:
+                continue
+
+            initial_hash = get_initial_observation(driver, url, content_type, selector)
+            if initial_hash:
                 HISTORY.append(f"Initial observation at {url}: {initial_hash}")
-                with open(os.path.join(folder_path, f"{hostname}_initial_observation.txt"), "w") as file:
-                    file.write(f"Initial observation at {url}: {initial_hash}")
-        except (NoSuchElementException, Exception) as e:
+
+                # Insert initial observation into MySQL database
+                connection = get_db_connection()
+                if connection:
+                    cursor = connection.cursor()
+                    insert_query = """
+                    INSERT INTO scraped_data (url, content_hash, change_detected)
+                    VALUES (%s, %s, %s)
+                    """
+                    cursor.execute(insert_query, (url, initial_hash, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+                    connection.commit()
+                    cursor.close()
+                    connection.close()
+                    logging.info(f"Initial observation logged for {url}")
+                else:
+                    logging.error("Failed to connect to database. Initial observation not logged.")
+
+        except Exception as e:
             HISTORY.append(f"Error accessing {url}: {e}")
-        
-        # Start a new thread for monitoring URLs
-        threading.Thread(target=monitor_urls, args=(storage_location, [url], scrape_interval, content_type, [STOP_THREADS], db_config)).start()
-    
-    return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."
-
-# Define a function to monitor URLs for changes
-def monitor_urls(storage_location, urls, scrape_interval, content_type, stop_scraping_flag, db_config):
-    global HISTORY
-    previous_hashes = {url: "" for url in urls}
-    
+            logging.error(f"Error accessing {url}: {e}")
+        finally:
+            driver.quit()
+
+    # Start the monitoring thread
+    monitor_thread = threading.Thread(
+        target=monitor_urls,
+        args=(storage_location, url_list, scrape_interval, content_type, selector),
+        daemon=True,
+    )
+    monitor_thread.start()
+    logging.info("Started scraping thread.")
+    return f"Started scraping {', '.join(url_list)} every {scrape_interval} minutes."
+
+# Function to stop scraping
+def stop_scraping() -> str:
+    """
+    Stops all ongoing scraping threads.
+    """
+    global STOP_THREADS
+    STOP_THREADS = True
+    HISTORY.append("Scraping stopped by user.")
+    logging.info("Scraping stop signal sent.")
+    return "Scraping has been stopped."
+
+# Function to display CSV content from MySQL
+def display_csv(storage_location: str, url: str) -> str:
+    """
+    Fetches and returns the scraped data for a given URL from the MySQL database.
+    """
     try:
-        cnx = mysql.connector.connect(**db_config)
-        cursor = cnx.cursor()
-        
-        with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager ().install()), options=Options()) as driver:
-            while not stop_scraping_flag[0]:
-                for url in urls:
-                    try:
-                        driver.get(url)
-                        time.sleep(2)  # Wait for the page to load
-                        if content_type == "text":
-                            current_content = driver.page_source
-                        elif content_type == "media":
-                            current_content = driver.find_elements(By.TAG_NAME, "img")
-                        else:
-                            current_content = driver.page_source
-                        current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
-                        
-                        if current_hash != previous_hashes[url]:
-                            previous_hashes[url] = current_hash
-                            date_time = datetime.datetime.now()
-                            HISTORY.append(f"Change detected at {url} on {date_time}")
-                            
-                            # Insert into MySQL
-                            add_change = ("INSERT INTO scraped_data "
-                                          "(url, content_hash, change_detected) "
-                                          "VALUES (%s, %s, %s)")
-                            data_change = (url, current_hash, date_time)
-                            cursor.execute(add_change, data_change)
-                            cnx.commit()
-                            
-                            logging.info(f"Change detected and logged for {url} at {date_time}")
-                    except (NoSuchElementException, Exception) as e:
-                        logging.error(f"Error accessing {url}: {e}")
-                time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
+        connection = get_db_connection()
+        if not connection:
+            return "Failed to connect to the database."
+
+        cursor = connection.cursor(dictionary=True)
+        query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC"
+        cursor.execute(query, (url,))
+        results = cursor.fetchall()
+
+        if not results:
+            return "No data available for the selected URL."
+
+        df = pd.DataFrame(results)
+        cursor.close()
+        connection.close()
+        return df.to_string(index=False)
     except Exception as e:
-        logging.error(f"Error in monitor_urls: {e}")
+        logging.error(f"Error fetching data for {url}: {e}")
+        return f"Error fetching data for {url}: {e}"
+
+# Function to generate RSS feed from MySQL data
+def generate_rss_feed(storage_location: str, url: str) -> str:
+    """
+    Generates an RSS feed for the latest changes detected on a given URL from the MySQL database.
+    """
+    try:
+        connection = get_db_connection()
+        if not connection:
+            return "Failed to connect to the database."
+
+        cursor = connection.cursor(dictionary=True)
+        query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC LIMIT 10"
+        cursor.execute(query, (url,))
+        results = cursor.fetchall()
+
+        if not results:
+            return "No changes detected to include in RSS feed."
+
+        # Create the root RSS element
+        rss = ET.Element("rss", version="2.0")
+        channel = ET.SubElement(rss, "channel")
+
+        # Add channel elements
+        title = ET.SubElement(channel, "title")
+        title.text = f"RSS Feed for {urlparse(url).hostname}"
+
+        link = ET.SubElement(channel, "link")
+        link.text = url
+
+        description = ET.SubElement(channel, "description")
+        description.text = "Recent changes detected on the website."
+
+        # Add items to the feed
+        for row in results:
+            item = ET.SubElement(channel, "item")
+
+            item_title = ET.SubElement(item, "title")
+            item_title.text = f"Change detected at {row['url']}"
+
+            item_link = ET.SubElement(item, "link")
+            item_link.text = row["url"]
+
+            item_description = ET.SubElement(item, "description")
+            item_description.text = f"Content changed on {row['change_detected']}"
+
+            pub_date = ET.SubElement(item, "pubDate")
+            pub_date.text = datetime.datetime.strptime(
+                str(row['change_detected']), "%Y-%m-%d %H:%M:%S"
+            ).strftime("%a, %d %b %Y %H:%M:%S +0000")
+
+        # Generate the XML string
+        rss_feed = ET.tostring(rss, encoding="utf-8", method="xml")
+        return rss_feed.decode("utf-8")
+    except Exception as e:
+        logging.error(f"Error generating RSS feed for {url}: {e}")
+        return f"Error generating RSS feed for {url}: {e}"
     finally:
         cursor.close()
-        cnx.close()
+        connection.close()
 
-# Define a function to stop scraping
-def stop_scraping():
-    global STOP_THREADS
-    STOP_THREADS = True
-    return "Scraping stopped."
+# Function to load the Mistral model
+def load_model():
+    """
+    Loads the Mistral model and tokenizer once and returns the pipeline.
+    """
+    model_name = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipe = pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            device=0 if torch.cuda.is_available() else -1,
+        )
+        logging.info("Mistral model loaded successfully.")
+        return pipe
+    except Exception as e:
+        logging.error(f"Error loading Mistral model: {e}")
+        return None
+
+# Load the model once at the start
+chat_pipeline = load_model()
+
+# Function to parse user commands
+def parse_command(message: str) -> tuple:
+    """
+    Parses the user message to identify if it contains a command.
+    Returns the command and its parameters if found, else (None, None).
+    """
+    # Define command patterns
+    patterns = {
+        "filter": r"filter\s+(?P<words>[\w\s,]+)\s+in\s+column\s+(?P<column>\w+)",
+        "sort": r"sort\s+(?P<column>\w+)\s+(?P<order>ascending|descending)",
+        "export": r"export\s+to\s+csv\s+as\s+(?P<filename>\w+\.csv)",
+        "log": r"log\s+action\s+(?P<action>.+)",
+    }
+
+    for command, pattern in patterns.items():
+        match = re.search(pattern, message, re.IGNORECASE)
+        if match:
+            params = match.groupdict()
+            return command, params
+
+    return None, None
+
+# Function to execute parsed commands
+def execute_command(command: str, params: dict) -> str:
+    """
+    Executes the corresponding function based on the command and parameters.
+    """
+    if command == "filter":
+        words = [word.strip() for word in params["words"].split(",")]
+        column = params["column"]
+        return filter_data(column, words)
+    elif command == "sort":
+        column = params["column"]
+        order = params["order"]
+        return sort_data(column, order)
+    elif command == "export":
+        filename = params["filename"]
+        return export_csv(filename)
+    elif command == "log":
+        action = params["action"]
+        return log_action(action)
+    else:
+        return "Unknown command."
+
+# Data Manipulation Functions
+def filter_data(column: str, words: list) -> str:
+    """
+    Filters the scraped data to include only rows where the specified column contains the given words.
+    Saves the filtered data to a new CSV file.
+    """
+    try:
+        latest_csv = get_latest_csv()
+        if not latest_csv:
+            return "No CSV files found to filter."
+
+        df = pd.read_csv(latest_csv)
+        # Create a regex pattern to match any of the words
+        pattern = '|'.join(words)
+        filtered_df = df[df[column].astype(str).str.contains(pattern, case=False, na=False)]
+
+        if filtered_df.empty:
+            return f"No records found with words {words} in column '{column}'."
+
+        # Save the filtered data to a new CSV
+        filtered_csv = latest_csv.replace(".csv", "_filtered.csv")
+        filtered_df.to_csv(filtered_csv, index=False)
+        logging.info(f"Data filtered on column '{column}' for words {words}.")
+        return f"Data filtered and saved to {filtered_csv}."
+    except Exception as e:
+        logging.error(f"Error filtering data: {e}")
+        return f"Error filtering data: {e}"
+
+def sort_data(column: str, order: str) -> str:
+    """
+    Sorts the scraped data based on the specified column and order.
+    Saves the sorted data to a new CSV file.
+    """
+    try:
+        latest_csv = get_latest_csv()
+        if not latest_csv:
+            return "No CSV files found to sort."
+
+        df = pd.read_csv(latest_csv)
+        ascending = True if order.lower() == "ascending" else False
+        sorted_df = df.sort_values(by=column, ascending=ascending)
+
+        # Save the sorted data to a new CSV
+        sorted_csv = latest_csv.replace(".csv", f"_sorted_{order.lower()}.csv")
+        sorted_df.to_csv(sorted_csv, index=False)
+        logging.info(f"Data sorted on column '{column}' in {order} order.")
+        return f"Data sorted and saved to {sorted_csv}."
+    except Exception as e:
+        logging.error(f"Error sorting data: {e}")
+        return f"Error sorting data: {e}"
+
+def export_csv(filename: str) -> str:
+    """
+    Exports the latest scraped data to a specified CSV filename.
+    """
+    try:
+        latest_csv = get_latest_csv()
+        if not latest_csv:
+            return "No CSV files found to export."
 
-# Define a function to generate RSS feed
-def generate_rss_feed(selected_url, db_config):
+        export_path = os.path.join(os.path.dirname(latest_csv), filename)
+        df = pd.read_csv(latest_csv)
+        df.to_csv(export_path, index=False)
+        logging.info(f"Data exported to {export_path}.")
+        return f"Data exported to {export_path}."
+    except Exception as e:
+        logging.error(f"Error exporting CSV: {e}")
+        return f"Error exporting CSV: {e}"
+
+def log_action(action: str) -> str:
+    """
+    Logs a custom action message to the MySQL database.
+    """
     try:
-        cnx = mysql.connector.connect(**db_config)
-        cursor = cnx.cursor(dictionary=True)
-        
-        query = ("SELECT content_hash, change_detected FROM scraped_data "
-                 "WHERE url = %s ORDER BY change_detected DESC LIMIT 10")
-        cursor.execute(query, (selected_url,))
-        
-        items = cursor.fetchall()
-        
-        rss_items = ""
-        for item in items:
-            rss_items += f"""
-            <item>
-                <title>Change Detected</title>
-                <link>{selected_url}</link>
-                <description>Change detected on {item['change_detected'].strftime('%Y-%m-%d %H:%M:%S')}</description>
-                <pubDate>{item['change_detected'].strftime('%a, %d %b %Y %H:%M:%S +0000')}</pubDate>
-            </item>
-            """
-        
-        rss_feed = f"""<?xml version="1.0" encoding="UTF-8"?>
-        <rss version="2.0">
-            <channel>
-                <title>RSS Feed for {selected_url}</title>
-                <link>{selected_url}</link>
-                <description>Latest changes detected on {selected_url}.</description>
-                {rss_items}
-            </channel>
-        </rss>"""
-        
+        connection = get_db_connection()
+        if not connection:
+            return "Failed to connect to the database."
+
+        cursor = connection.cursor()
+        insert_query = """
+        INSERT INTO action_logs (action)
+        VALUES (%s)
+        """
+        cursor.execute(insert_query, (action,))
+        connection.commit()
         cursor.close()
-        cnx.close()
-        return rss_feed
-    except mysql.connector.Error as err:
-        logging.error(f"Error generating RSS feed: {err}")
-        return "Failed to generate RSS feed."
+        connection.close()
+
+        HISTORY.append(f"User Action Logged: {action}")
+        logging.info(f"Action logged: {action}")
+        return f"Action logged: {action}"
+    except Exception as e:
+        logging.error(f"Error logging action: {e}")
+        return f"Error logging action: {e}"
+
+def get_latest_csv() -> str:
+    """
+    Retrieves the latest CSV file from the storage directory based on modification time.
+    """
+    try:
+        storage_dirs = [d for d in os.listdir(DEFAULT_FILE_PATH) if os.path.isdir(os.path.join(DEFAULT_FILE_PATH, d))]
+        if not storage_dirs:
+            return None
+
+        latest_csv = None
+        latest_time = 0
+        for dir_name in storage_dirs:
+            dir_path = os.path.join(DEFAULT_FILE_PATH, dir_name)
+            csv_files = [f for f in os.listdir(dir_path) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
+            for csv_file in csv_files:
+                csv_path = os.path.join(dir_path, csv_file)
+                mod_time = os.path.getmtime(csv_path)
+                if mod_time > latest_time:
+                    latest_time = mod_time
+                    latest_csv = csv_path
+        return latest_csv
+    except Exception as e:
+        logging.error(f"Error retrieving latest CSV: {e}")
+        return None
+
+# Chat Response Function with Dynamic Command Handling
+def respond(
+    message: str,
+    history: list,
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
+) -> str:
+    """
+    Generates a response using the Mistral model based on the user's message and history.
+    Additionally, handles dynamic commands to interact with individual components.
+    """
+    if chat_pipeline is None:
+        return "Error: Chat model is not loaded."
 
-# Define a function to handle messages
-def handle_message(message, chat_history, system_message, max_tokens, temperature, top_p):
-    chat_history.append((message, system_message))
-    response = f"Received message: {message}"
-    return chat_history, response
+    try:
+        # Check if the message contains a command
+        command, params = parse_command(message)
+        if command:
+            # Execute the corresponding function
+            response = execute_command(command, params)
+        else:
+            # Generate a regular response using the model
+            prompt = (
+                f"System: {system_message}\n"
+                f"History: {history}\n"
+                f"User: {message}\n"
+                f"Assistant:"
+            )
+            response = chat_pipeline(
+                prompt,
+                max_length=max_tokens,
+                temperature=temperature,
+                top_p=top_p,
+                num_return_sequences=1,
+            )[0]["generated_text"]
+
+            # Extract the assistant's reply
+            response = response.split("Assistant:")[-1].strip()
+        return response
+    except Exception as e:
+        logging.error(f"Error generating response: {e}")
+        return "Error generating response."
+
+# Function to load the Mistral model
+def load_model():
+    """
+    Loads the Mistral model and tokenizer once and returns the pipeline.
+    """
+    model_name = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        pipe = pipeline(
+            "text-generation",
+            model=model,
+            tokenizer=tokenizer,
+            device=0 if torch.cuda.is_available() else -1,
+        )
+        logging.info("Mistral model loaded successfully.")
+        return pipe
+    except Exception as e:
+        logging.error(f"Error loading Mistral model: {e}")
+        return None
+
+# Load the model once at the start
+chat_pipeline = load_model()
 
 # Define the Gradio interface
-def create_interface():
+def create_interface() -> gr.Blocks:
+    """
+    Defines and returns the Gradio interface for the application.
+    """
     with gr.Blocks() as demo:
+        gr.Markdown("# All-in-One Scraper, Database, and RSS Feeder")
+
         with gr.Row():
             with gr.Column():
-                message = gr.Textbox(label="Message")
-                system_message = gr.Textbox(value="You are a helpful assistant.", label="System message")
-                max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
-                temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-                top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
-                storage_location = gr.Textbox(value="scraped_data", label="Storage Location")
-                urls = gr.Textbox(label="URLs (comma separated)")
-                scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
-                content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
+                # Scraping Controls
+                storage_location = gr.Textbox(
+                    value=DEFAULT_FILE_PATH, label="Storage Location"
+                )
+                urls = gr.Textbox(
+                    label="URLs (comma separated)",
+                    placeholder="https://example.com, https://anotherexample.com",
+                )
+                scrape_interval = gr.Slider(
+                    minimum=1,
+                    maximum=60,
+                    value=5,
+                    step=1,
+                    label="Scrape Interval (minutes)",
+                )
+                content_type = gr.Radio(
+                    choices=["text", "media", "both"],
+                    value="text",
+                    label="Content Type",
+                )
+                selector = gr.Textbox(
+                    label="CSS Selector for Media (Optional)",
+                    placeholder="e.g., img.main-image",
+                )
                 start_button = gr.Button("Start Scraping")
                 stop_button = gr.Button("Stop Scraping")
-                csv_output = gr.Textbox(label="CSV Output", interactive=False)
-                 model_name_input = gr.Textbox(value="default_model", label="Model Name")
-                gpu_layers_input = gr.Slider(minimum=0, maximum=8, value=2, step=1, label="GPU Layers")
+                status_output = gr.Textbox(
+                    label="Status Output", interactive=False, lines=2
+                )
+
             with gr.Column():
+                # Chat Interface
                 chat_history = gr.Chatbot(label="Chat History")
-                response_box = gr.Textbox(label="Response")
+                with gr.Row():
+                    message = gr.Textbox(label="Message", placeholder="Type your message here...")
+                system_message = gr.Textbox(
+                    value="You are a helpful assistant.", label="System message"
+                )
+                max_tokens = gr.Slider(
+                    minimum=1,
+                    maximum=2048,
+                    value=512,
+                    step=1,
+                    label="Max new tokens",
+                )
+                temperature = gr.Slider(
+                    minimum=0.1,
+                    maximum=4.0,
+                    value=0.7,
+                    step=0.1,
+                    label="Temperature",
+                )
+                top_p = gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.95,
+                    step=0.05,
+                    label="Top-p (nucleus sampling)",
+                )
+                response_box = gr.Textbox(label="Response", interactive=False, lines=2)
+
+        with gr.Row():
+            with gr.Column():
+                # CSV Display Controls
+                selected_url_csv = gr.Textbox(
+                    label="Select URL for CSV Content",
+                    placeholder="https://example.com",
+                )
+                csv_button = gr.Button("Display CSV Content")
+                csv_content_output = gr.Textbox(
+                    label="CSV Content Output", interactive=False, lines=10
+                )
+
+            with gr.Column():
+                # RSS Feed Generation Controls
+                selected_url_rss = gr.Textbox(
+                    label="Select URL for RSS Feed",
+                    placeholder="https://example.com",
+                )
+                rss_button = gr.Button("Generate RSS Feed")
+                rss_output = gr.Textbox(
+                    label="RSS Feed Output", interactive=False, lines=20
+                )
 
         # Connect buttons to their respective functions
         start_button.click(
-            fn=lambda storage, urls, interval, ctype: start_scraping(
-                storage, urls.split(", "), interval, ctype, db_config
-            ),
-            inputs=[storage_location, urls, scrape_interval, content_type],
-            outputs=csv_output
+            fn=start_scraping,
+            inputs=[
+                storage_location,
+                urls,
+                scrape_interval,
+                content_type,
+                selector,
+            ],
+            outputs=status_output,
         )
-        stop_button.click(stop_scraping, outputs=csv_output)
 
-        # Connect message submission to the chat interface
-        message.submit(handle_message, inputs=[message, chat_history, system_message, max_tokens, temperature, top_p], outputs=[chat_history, response_box])
+        stop_button.click(fn=stop_scraping, outputs=status_output)
 
-        # Add a button to display the CSV content for a selected URL
-        with gr.Row():
-            selected_url = gr.Textbox(label="Select URL for CSV Content")
-            csv_button = gr.Button("Display CSV Content")
-            csv_output = gr.Textbox(label="CSV Content Output", interactive=False)
-        csv_button.click(display_csv, inputs=[selected_url], outputs=csv_output)
+        csv_button.click(
+            fn=display_csv,
+            inputs=[storage_location, selected_url_csv],
+            outputs=csv_content_output,
+        )
 
-        # Add a button to display the RSS feed for a selected URL
-        with gr.Row():
-            selected_url = gr.Textbox(label="Select URL for RSS Feed")
-            rss_button = gr.Button("Generate RSS Feed")
-            rss_output = gr.Textbox(label="RSS Feed Output", interactive=False)
         rss_button.click(
-            generate_rss_feed,
-            inputs=[selected_url, gr.State(db_config)],
-            outputs=rss_output
+            fn=generate_rss_feed,
+            inputs=[storage_location, selected_url_rss],
+            outputs=rss_output,
+        )
+
+        # Connect message submission to the chat interface
+        def update_chat(message_input, history, system_msg, max_toks, temp, top_p_val):
+            if not message_input.strip():
+                return history, "Please enter a message."
+
+            response = respond(
+                message_input,
+                history,
+                system_msg,
+                max_toks,
+                temp,
+                top_p_val,
+            )
+            history.append((message_input, response))
+            return history, response
+
+        message.submit(
+            update_chat,
+            inputs=[
+                message,
+                chat_history,
+                system_message,
+                max_tokens,
+                temperature,
+                top_p,
+            ],
+            outputs=[chat_history, response_box],
         )
 
     return demo
 
-# Initialize the database
-initialize_database(db_config)
+# Initialize database on script start
+initialize_database()
 
-# Launch the Gradio interface
-demo = create_interface()
-demo.launch()
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.launch()