app.py

limport datetimeimport osimport csvimport timeimport hashlibimport loggingfrom collections import defaultdictimport mysql.connectorimport threadingfrom urllib.parse import urlparseimport gradio as grfrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as   



 ECfrom selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutExceptionfrom selenium.webdriver.chrome.service   



 import Servicefrom selenium.webdriver.chrome.options import Optionsfrom webdriver_manager.chrome import ChromeDriverManager   



from huggingface_hub import InferenceClient, loginfrom transformers import AutoTokenizer, AutoModelForCausalLM, pipelineimport randomimport yamlimport torchimport pandas as pdimport xml.etree.ElementTree as ETimport reimport spacyimport unittestfrom dotenv import load_dotenvimport nltk# Initialize NLTK resources (you may need to download these)

nltk.download('punkt')nltk.download('averaged_perceptron_tagger')

nltk.download('maxent_ne_chunker')

nltk.download('words')   



# Load spaCy model

nlp = spacy.load("en_core_web_sm")   



# Dictionary to store model loading functions

model_loaders = {

    "Falcon": lambda: load_model("tiiuae/falcon-7b"),

    "Flan-T5": lambda: load_model("google/flan-t5-xl"),

    "Flan-T5-Small": lambda: load_model("google/flan-t5-small")  # Add a smaller model

}# Load environment variables from .env file

load_dotenv()



HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")if not HUGGINGFACE_TOKEN:

    raise ValueError("HUGGINGFACE_TOKEN is not set in the environment variables.")



login(token=HUGGINGFACE_TOKEN, add_to_git_credential=True)# Configure logging

logging.basicConfig(

    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"

)# Define constants

DEFAULT_FILE_PATH = "scraped_data"

PURPOSE = (

    "You monitor urls. You log what you observe. You seek any changes on them since your last observation. "

    "Anything new gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."

)# Global variables for task management

HISTORY = []

CURRENT_TASK = None

STOP_THREADS = False  # Flag to stop scraping threads# Database Pooling Configuration

DB_POOL_NAME = "mypool"

DB_POOL_SIZE = 5  # Adjust based on expected loadtry:

    dbconfig = {

        "host": os.getenv("DB_HOST"),

        "user": os.getenv("DB_USER"),

        "password": os.getenv("DB_PASSWORD"),

        "database": os.getenv("DB_NAME"),

    }

    connection_pool = mysql.connector.pooling.MySQLConnectionPool(

        pool_name=DB_POOL_NAME,

        pool_size=DB_POOL_SIZE,

        pool_reset_session=True,

        **dbconfig

    )

    logging.info("Database connection pool created successfully.")except mysql.connector.Error as err:

    logging.warning(f"Database connection pool creation failed: {err}")

    connection_pool = None  # Will use CSV as fallback# Function to get a database connection from the pooldef get_db_connection():

    """

    Retrieves a connection from the pool. Returns None if pool is not available.

    """

    if connection_pool:

        try:

            connection = connection_pool.get_connection()

            if connection.is_connected():

                return connection

        except mysql.connector.Error as err:

            logging.error(f"Error getting connection from pool: {err}")

    return None# Initialize Database: Create tables and indexesdef initialize_database():

    """

    Initializes the database by creating necessary tables and indexes if they do not exist.

    """

    connection = get_db_connection()

    if connection is None:

        logging.info("Database initialization skipped. Using CSV storage.")

        return



    cursor = connection.cursor()

    try:

        # Create table for scraped data

        create_scraped_data_table = """

        CREATE TABLE IF NOT EXISTS scraped_data (

            id INT AUTO_INCREMENT PRIMARY KEY,

            url VARCHAR(255) NOT NULL,

            content_hash VARCHAR(64) NOT NULL,

            change_detected DATETIME NOT NULL

        )

        """

        cursor.execute(create_scraped_data_table)

        logging.info("Table 'scraped_data' is ready.")



        # Create indexes for performance

        create_index_url = "CREATE INDEX IF NOT EXISTS idx_url ON scraped_data(url)"

        create_index_change = "CREATE INDEX IF NOT EXISTS idx_change_detected ON scraped_data(change_detected)"

        cursor.execute(create_index_url)

        cursor.execute(create_index_change)

        logging.info("Indexes on 'url' and 'change_detected' columns created.")



        # Create table for action logs

        create_action_logs_table = """

        CREATE TABLE IF NOT EXISTS action_logs (

            id INT AUTO_INCREMENT PRIMARY KEY,

            action VARCHAR(255) NOT NULL,

            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP

        )

        """

        cursor.execute(create_action_logs_table)

        logging.info("Table 'action_logs' is ready.")



    except mysql.connector.Error as err:

        logging.error(f"Error initializing database: {err}")

    finally:

        cursor.close()

        connection.close()

        logging.info("Database initialization complete.")# Function to create WebDriverdef create_driver(options: Options) -> webdriver.Chrome:

    """

    Initializes and returns a Selenium Chrome WebDriver instance.

    """

    try:

        driver = webdriver.Chrome(

            service=Service(ChromeDriverManager().install()), options=options

        )

        logging.info("ChromeDriver initialized successfully.")

        return driver

    except Exception as exception:

        logging.error(f"Error initializing ChromeDriver: {exception}")

        return None# Function to log changes to CSVdef log_to_csv(storage_location: str, url: str, content_hash: str, change_detected: str):

    """

    Logs the change to a CSV file in the storage_location.

    """

    try:

        os.makedirs(storage_location, exist_ok=True)

        csv_file_path = os.path.join(storage_location, f"{urlparse(url).hostname}_changes.csv")

        file_exists = os.path.isfile(csv_file_path)



        with open(csv_file_path, "a", newline="", encoding="utf-8") as csvfile:

            fieldnames = ["date", "time", "url", "content_hash", "change"]

            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

            if not file_exists:

                writer.writeheader()

            writer.writerow(

                {

                    "date": change_detected.split()[0],

                    "time": change_detected.split()[1],

                    "url": url,

                    "content_hash": content_hash,

                    "change": "Content changed",

                }

            )

        logging.info(f"Change detected at {url} on {change_detected} and logged to CSV.")

    except Exception as e:

        logging.error(f"Error logging data to CSV: {e}")# Function to get initial observationdef get_initial_observation(

    driver: webdriver.Chrome, url: str, content_type: str, selector: str = None) -> str:

    """

    Retrieves the initial content from the URL and returns its MD5 hash.

    """

    try:

        driver.get(url)

        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))

        time.sleep(2)  # Additional wait for dynamic content



        if content_type == "text":

            initial_content = driver.page_source

        elif content_type == "media":

            if selector:

                try:

                    elements = WebDriverWait(driver, 5).until(

                        EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))

                    )

                    initial_content = [element.get_attribute("src") for element in elements]

                except TimeoutException:                     logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")

                    initial_content = []

            else:

                elements = driver.find_elements(By.TAG_NAME, "img")

                initial_content = [element.get_attribute("src") for element in elements]

        else:

            initial_content = driver.page_source



        initial_hash = hashlib.md5(str(initial_content).encode("utf-8")).hexdigest()

        logging.info(f"Initial hash for {url}: {initial_hash}")

        return initial_hash

    except Exception as exception:

        logging.error(f"Error accessing {url}: {exception}")

        return None# Function to monitor URLs for changesdef monitor_urls(

    storage_location: str,

    urls: list,

    scrape_interval: int,

    content_type: str,

    selector: str = None,

    progress: gr.Progress = None):

    """

    Monitors the specified URLs for changes and logs any detected changes to the database or CSV.

    """

    global HISTORY, STOP_THREADS

    previous_hashes = {url: "" for url in urls}



    options = Options()

    options.add_argument("--headless")

    options.add_argument("--no-sandbox")

    options.add_argument("--disable-dev-shm-usage")   







    driver = create_driver(options)

    if driver is None:

        logging.error("WebDriver could not be initialized. Exiting monitor.")

        return



    try:

        while not STOP_THREADS:

            for url in urls:

                if STOP_THREADS:

                    break

                try:

                    driver.get(url)

                    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))

                    time.sleep(2)  # Additional wait for dynamic content



                    if content_type == "text":

                        current_content = driver.page_source

                    elif content_type == "media":

                        if selector:

                            try:

                                elements = WebDriverWait(driver, 5).until(

                                    EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))

                                )

                                current_content = [element.get_attribute("src") for element in elements]

                            except TimeoutException:

                                logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")

                                current_content = []

                        else:

                            elements = driver.find_elements(By.TAG_NAME, "img")

                            current_content = [element.get_attribute("src") for element in elements]

                    else:

                        current_content = driver.page_source



                    current_hash = hashlib.md5(str(current_content).encode("utf-8")).hexdigest()

                    if current_hash != previous_hashes[url]:

                        previous_hashes[url] = current_hash

                        date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

                        HISTORY.append(f"Change detected at {url} on {date_time_str}")



                        # Attempt to log to database

                        connection = get_db_connection()

                        if connection:

                            try:

                                cursor = connection.cursor()

                                insert_query = """

                                INSERT INTO scraped_data (url, content_hash, change_detected)

                                VALUES (%s, %s, %s)

                                """

                                cursor.execute(insert_query, (url, current_hash, date_time_str))

                                connection.commit()

                                logging.info(f"Change detected at {url} on {date_time_str} and logged to database.")

                            except mysql.connector.Error as err:

                                logging.error(f"Error inserting data into database: {err}")

                                # Fallback to CSV

                                log_to_csv(storage_location, url, current_hash, date_time_str)

                            finally:

                                cursor.close()

                                connection.close()

                        else:

                            # Fallback to CSV

                            log_to_csv(storage_location, url, current_hash, date_time_str)



                        # Update progress

                        if progress:

                            progress(1)

                except (

                    NoSuchElementException,

                    StaleElementReferenceException,

                    TimeoutException,

                    Exception,

                ) as e:

                    logging.error(f"Error accessing {url}: {e}")

                    if progress:

                        progress(1)

            time.sleep(scrape_interval * 60)  # Wait for the next scrape interval

    finally:

        driver.quit()

        logging.info("ChromeDriver session ended.")# Function to start scrapingdef start_scraping(

    storage_location: str,

    urls: str,

    scrape_interval: int,

    content_type: str,

    selector: str = None,

    progress: gr.Progress = None) -> str:

    """

    Starts the scraping process in a separate thread with progress indication.

    """

    global CURRENT_TASK, HISTORY, STOP_THREADS



    if STOP_THREADS:

        STOP_THREADS = False  # Reset the flag if previously stopped



    url_list = [url.strip() for url in urls.split(",") if url.strip()]

    CURRENT_TASK = f"Monitoring URLs: {', '.join(url_list)}"

    HISTORY.append(f"Task started: {CURRENT_TASK}")

    logging.info(f"Task started: {CURRENT_TASK}")



    # Initialize database tables

    initialize_database()



    # Log initial observations

    def log_initial_observations():

        options = Options()

        options.add_argument("--headless")

        options.add_argument("--no-sandbox")

        options.add_argument("--disable-dev-shm-usage")   







        driver = create_driver(options)

        if driver is None:

            return



        for url in url_list:

            if STOP_THREADS:

                break

            try:

                initial_hash = get_initial_observation(driver, url, content_type, selector)

                if initial_hash:

                    date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

                    HISTORY.append(f"Initial observation at {url}: {initial_hash}")



                    # Attempt to log to database

                    connection = get_db_connection()

                    if connection:

                        try:

                            cursor = connection.cursor()

                            insert_query = """

                            INSERT INTO scraped_data (url, content_hash, change_detected)

                            VALUES (%s, %s, %s)

                            """

                            cursor.execute(insert_query, (url, initial_hash, date_time_str))

                            connection.commit()

                            logging.info(f"Initial observation logged for {url} in database.")

                        except mysql.connector.Error as err:

                            logging.error(f"Error inserting initial observation into database: {err}")

                            # Fallback to CSV

                            log_to_csv(storage_location, url, initial_hash, date_time_str)

                        finally:

                            cursor.close()

                            connection.close()

                    else:

                        # Fallback to CSV

                        log_to_csv(storage_location, url, initial_hash, date_time_str)

            except Exception as e:

                HISTORY.append(f"Error accessing {url}: {e}")

                logging.error(f"Error accessing {url}: {e}")

        driver.quit()



    # Start logging initial observations

    initial_thread = threading.Thread(target=log_initial_observations, daemon=True)

    initial_thread.start()



    # Start the monitoring thread with progress

    monitor_thread = threading.Thread(

        target=monitor_urls,

        args=(storage_location, url_list, scrape_interval, content_type, selector, progress),

        daemon=True,

    )

    monitor_thread.start()

    logging.info("Started scraping thread.")

    return f"Started scraping {', '.join(url_list)} every {scrape_interval} minutes."

# Function to stop scraping
def stop_scraping() -> str:
    """
    Stops all ongoing scraping threads.
    """
    global STOP_THREADS
    STOP_THREADS = True
    HISTORY.append("Scraping stopped by user.")
    logging.info("Scraping stop signal sent.")
    return "Scraping has been stopped."

# Function to display CSV content from MySQL or CSV
def display_csv(storage_location: str, url: str) -> str:
    """
    Fetches and returns the scraped data for a given URL from the MySQL database or CSV.
    """
    try:
        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC"
                cursor.execute(query, (url,))
                results = cursor.fetchall()

                if not results:
                    return "No data available for the selected URL."

                df = pd.DataFrame(results)
                cursor.close()
                connection.close()
                return df.to_string(index=False)
            except mysql.connector.Error as err:
                logging.error(f"Error fetching data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Fetching data from CSV.")

        # Fallback to CSV
        hostname = urlparse(url).hostname
        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
        if os.path.exists(csv_path):
            df = pd.read_csv(csv_path)
            return df.to_string(index=False)
        else:
            return "No data available."

    except Exception as e:
        logging.error(f"Error fetching data for {url}: {e}")
        return f"Error fetching data for {url}: {e}"

# Function to generate RSS feed from MySQL or CSV data
def generate_rss_feed(storage_location: str, url: str) -> str:
    """
    Generates an RSS feed for the latest changes detected on a given URL from the MySQL database or CSV.
    """
    try:
        connection = get_db_connection()
        rss_feed = ""

        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC LIMIT 10"
                cursor.execute(query, (url,))
                results = cursor.fetchall()

                if not results:
                    return "No changes detected to include in RSS feed."

                # Create the root RSS element
                rss = ET.Element("rss", version="2.0")
                channel = ET.SubElement(rss, "channel")

                # Add channel elements
                title = ET.SubElement(channel, "title")
                title.text = f"RSS Feed for {urlparse(url).hostname}"

                link = ET.SubElement(channel, "link")
                link.text = url

                description = ET.SubElement(channel, "description")
                description.text = "Recent changes detected on the website."

                # Add items to the feed
                for row in results:
                    item = ET.SubElement(channel, "item")

                    item_title = ET.SubElement(item, "title")
                    item_title.text = f"Change detected at {row['url']}"

                    item_link = ET.SubElement(item, "link")
                    item_link.text = row["url"]

                    item_description = ET.SubElement(item, "description")
                    item_description.text = f"Content changed on {row['change_detected']}"

                    pub_date = ET.SubElement(item, "pubDate")
                    pub_date.text = datetime.datetime.strptime(
                        str(row['change_detected']), "%Y-%m-%d %H:%M:%S"
                    ).strftime("%a, %d %b %Y %H:%M:%S +0000")

                # Generate the XML string
                rss_feed = ET.tostring(rss, encoding="utf-8", method="xml").decode("utf-8")
                cursor.close()
                connection.close()
                return rss_feed
            except mysql.connector.Error as err:
                logging.error(f"Error fetching data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Generating RSS feed from CSV.")

        # Fallback to CSV
        hostname = urlparse(url).hostname
        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
        if os.path.exists(csv_path):
            df = pd.read_csv(csv_path).tail(10)
            if df.empty:
                return "No changes detected to include in RSS feed."

            # Create the root RSS element
            rss = ET.Element("rss", version="2.0")
            channel = ET.SubElement(rss, "channel")

            # Add channel elements
            title = ET.SubElement(channel, "title")
            title.text = f"RSS Feed for {hostname}"

            link = ET.SubElement(channel, "link")
            link.text = url

            description = ET.SubElement(channel, "description")
            description.text = "Recent changes detected on the website."

            # Add items to the feed
            for _, row in df.iterrows():
                item = ET.SubElement(channel, "item")

                item_title = ET.SubElement(item, "title")
                item_title.text = f"Change detected at {row['url']}"

                item_link = ET.SubElement(item, "link")
                item_link.text = row["url"]

                item_description = ET.SubElement(item, "description")
                item_description.text = f"Content changed on {row['date']} at {row['time']}"

                pub_date = ET.SubElement(item, "pubDate")
                pub_date.text = datetime.datetime.strptime(
                    f"{row['date']} {row['time']}", "%Y-%m-%d %H:%M:%S"
                ).strftime("%a, %d %b %Y %H:%M:%S +0000")

            # Generate the XML string
            rss_feed = ET.tostring(rss, encoding="utf-8", method="xml").decode("utf-8")
            return rss_feed
        else:
            return "No data available."

    except Exception as e:
        logging.error(f"Error generating RSS feed for {url}: {e}")
        return f"Error generating RSS feed for {url}: {e}"

# Function to parse user commands using spaCy
def parse_command(message: str) -> tuple:
    """
    Parses the user message using spaCy to identify if it contains a command.
    Returns the command and its parameters if found, else (None, None).
    """
    doc = nlp(message.lower())
    command = None
    params = {}

    # Define command patterns
    if "filter" in message.lower():
        # Example: "Filter apples, oranges in column Description"
        match = re.search(r"filter\s+([\w\s,]+)\s+in\s+column\s+(\w+)", message, re.IGNORECASE)
        if match:
            words = [word.strip() for word in match.group(1).split(",")]
            column = match.group(2)
            command = "filter"
            params = {"words": words, "column": column}

    elif "sort" in message.lower():
        # Example: "Sort Price ascending"
        match = re.search(r"sort\s+(\w+)\s+(ascending|descending)", message, re.IGNORECASE)
        if match:
            column = match.group(1)
            order = match.group(2)
            command = "sort"
            params = {"column": column, "order": order}

    elif "export to csv as" in message.lower():
        # Example: "Export to CSV as filtered_data.csv"
        match = re.search(r"export\s+to\s+csv\s+as\s+([\w\-]+\.csv)", message, re.IGNORECASE)
        if match:
            filename = match.group(1)
            command = "export"
            params = {"filename": filename}

    elif "log action" in message.lower():
        # Example: "Log action Filtered data for specific fruits"
        match = re.search(r"log\s+action\s+(.+)", message, re.IGNORECASE)
        if match:
            action = match.group(1)
            command = "log"
            params = {"action": action}

    return command, params

# Function to execute parsed commands
def execute_command(command: str, params: dict) -> str:
    """
    Executes the corresponding function based on the command and parameters.
    """
    if command == "filter":
        words = params["words"]
        column = params["column"]
        return filter_data(column, words)
    elif command == "sort":
        column = params["column"]
        order = params["order"]
        return sort_data(column, order)
    elif command == "export":
        filename = params["filename"]
        return export_csv(filename)
    elif command == "log":
        action = params["action"]
        return log_action(action)
    else:
        return "Unknown command."

# Data Manipulation Functions
def filter_data(column: str, words: list) -> str:
    """
    Filters the scraped data to include only rows where the specified column contains the given words.
    Saves the filtered data to a new CSV file.
    """
    try:
        storage_location = DEFAULT_FILE_PATH

        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                # Fetch all data
                query = "SELECT * FROM scraped_data"
                cursor.execute(query)
                results = cursor.fetchall()

                if not results:
                    return "No data available to filter."

                df = pd.DataFrame(results)
                # Create a regex pattern to match any of the words
                pattern = '|'.join(words)
                if column not in df.columns:
                    return f"Column '{column}' does not exist in the data."

                filtered_df = df[df[column].astype(str).str.contains(pattern, case=False, na=False)]

                if filtered_df.empty:
                    return f"No records found with words {words} in column '{column}'."

                # Save the filtered data to a new CSV
                timestamp = int(time.time())
                filtered_csv = os.path.join(storage_location, f"filtered_data_{timestamp}.csv")
                filtered_df.to_csv(filtered_csv, index=False)
                logging.info(f"Data filtered on column '{column}' for words {words}.")
                return f"Data filtered and saved to {filtered_csv}."
            except mysql.connector.Error as err:
                logging.error(f"Error fetching data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Filtering data from CSV.")

        # Fallback to CSV
        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
        if not csv_files:
            return "No CSV files found to filter."

        # Assume the latest CSV is the target
        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
        df = pd.read_csv(latest_csv)

        if column not in df.columns:
            return f"Column '{column}' does not exist in the data."

        filtered_df = df[df[column].astype(str).str.contains('|'.join(words), case=False, na=False)]

        if filtered_df.empty:
            return f"No records found with words {words} in column '{column}'."

        # Save the filtered data to a new CSV
        timestamp = int(time.time())
        filtered_csv = latest_csv.replace(".csv", f"_filtered_{timestamp}.csv")
        filtered_df.to_csv(filtered_csv, index=False)
        logging.info(f"Data filtered on column '{column}' for words {words}.")
        return f"Data filtered and saved to {filtered_csv}."
    except Exception as e:
        logging.error(f"Error filtering data: {e}")
        return f"Error filtering data: {e}"

def sort_data(column: str, order: str) -> str:
    """
    Sorts the scraped data based on the specified column and order.
    Saves the sorted data to a new CSV file.
    """
    try:
        storage_location = DEFAULT_FILE_PATH

        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                # Fetch all data
                query = "SELECT * FROM scraped_data"
                cursor.execute(query)
                results = cursor.fetchall()

                if not results:
                    return "No data available to sort."

                df = pd.DataFrame(results)
                if column not in df.columns:
                    return f"Column '{column}' does not exist in the data."

                ascending = True if order.lower() == "ascending" else False
                sorted_df = df.sort_values(by=column, ascending=ascending)

                # Save the sorted data to a new CSV
                timestamp = int(time.time())
                sorted_csv = os.path.join(storage_location, f"sorted_data_{column}_{order.lower()}_{timestamp}.csv")
                sorted_df.to_csv(sorted_csv, index=False)
                logging.info(f"Data sorted on column '{column}' in {order} order.")
                return f"Data sorted and saved to {sorted_csv}."
            except mysql.connector.Error as err:
                logging.error(f"Error fetching data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Sorting data from CSV.")

        # Fallback to CSV
        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
        if not csv_files:
            return "No CSV files found to sort."

        # Assume the latest CSV is the target
        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
        df = pd.read_csv(latest_csv)

        if column not in df.columns:
            return f"Column '{column}' does not exist in the data."

        ascending = True if order.lower() == "ascending" else False
        sorted_df = df.sort_values(by=column, ascending=ascending)

        # Save the sorted data to a new CSV
        timestamp = int(time.time())
        sorted_csv = latest_csv.replace(".csv", f"_sorted_{order.lower()}_{timestamp}.csv")
        sorted_df.to_csv(sorted_csv, index=False)
        logging.info(f"Data sorted on column '{column}' in {order} order.")
        return f"Data sorted and saved to {sorted_csv}."
    except Exception as e:
        logging.error(f"Error sorting data: {e}")
        return f"Error sorting data: {e}"

def export_csv(filename: str) -> str:
    """
    Exports the latest scraped data to a specified CSV filename.
    """
    try:
        storage_location = DEFAULT_FILE_PATH

        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                # Fetch all data
                query = "SELECT * FROM scraped_data"
                cursor.execute(query)
                results = cursor.fetchall()

                if not results:
                    return "No data available to export."

                df = pd.DataFrame(results)
                export_path = os.path.join(storage_location, filename)
                df.to_csv(export_path, index=False)
                logging.info(f"Data exported to {export_path}.")
                return f"Data exported to {export_path}."
            except mysql.connector.Error as err:
                logging.error(f"Error exporting data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Exporting data from CSV.")

        # Fallback to CSV
        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
        if not csv_files:
            return "No CSV files found to export."

        # Assume the latest CSV is the target
        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
        df = pd.read_csv(latest_csv)
        export_path = os.path.join(storage_location, filename)
        df.to_csv(export_path, index=False)
        logging.info(f"Data exported to {export_path}.")
        return f"Data exported to {export_path}."
    except Exception as e:
        logging.error(f"Error exporting CSV: {e}")
        return f"Error exporting CSV: {e}"

def log_action(action: str) -> str:
    """
    Logs a custom action message to the MySQL database or CSV.
    """
    try:
        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor()
                insert_query = """
                INSERT INTO action_logs (action)
                VALUES (%s)
                """
                cursor.execute(insert_query, (action,))
                connection.commit()
                logging.info(f"Action logged in database: {action}")
                cursor.close()
                connection.close()
                return f"Action logged: {action}"
            except mysql.connector.Error as err:
                logging.error(f"Error logging action to database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Logging action to CSV.")

        # Fallback to CSV
        storage_location = DEFAULT_FILE_PATH
        try:
            os.makedirs(storage_location, exist_ok=True)
            csv_file_path = os.path.join(storage_location, "action_logs.csv")
            file_exists = os.path.isfile(csv_file_path)

            with open(csv_file_path, "a", newline="", encoding="utf-8") as csvfile:
                fieldnames = ["timestamp", "action"]
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                if not file_exists:
                    writer.writeheader()
                writer.writerow(
                    {
                        "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                        "action": action,
                    }
                )
            logging.info(f"Action logged to CSV: {action}")
            return f"Action logged: {action}"
        except Exception as e:
            logging.error(f"Error logging action to CSV: {e}")
            return f"Error logging action: {e}"
    except Exception as e:
        logging.error(f"Error logging action: {e}")
        return f"Error logging action: {e}"

# Function to get the latest CSV file based on modification time
def get_latest_csv() -> str:
    """
    Retrieves the latest CSV file from the storage directory based on modification time.
    """
    try:
        storage_location = "/home/users/app/scraped_data"
        csv_files = [f for f in os.listdir(storage_location) if f.endswith(".csv")]
        if not csv_files:
            return None

        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
        return latest_csv
    except Exception as e:
        logging.error(f"Error retrieving latest CSV: {e}")
        return None

def respond(
    message: str,
    history: list,
    system_message: str,
    max_tokens: int,
    temperature: float,
    top_p: float,
) -> str:
    """
    Generates a response using OpenLlamaForCausalLM.
    """
    try:
        # Check if the message contains a command
        command, params = parse_command(message)
        if command:
            # Execute the corresponding function
            response = execute_command(command, params)
        else:
            # Generate a regular response using OpenLlama
            prompt = (
                f"System: {system_message}\n"
                f"History: {history}\n"
                f"User: {message}\n"
                f"Assistant:"
            )
            response = openllama_pipeline(
                prompt,
                max_length=max_tokens,
                temperature=temperature,
                top_p=top_p,
            )[0]["generated_text"]


            # Extract the assistant's reply
            response = response.split("Assistant:")[-1].strip()
        return response
    except Exception as e:
        logging.error(f"Error generating response: {e}")
        return "Error generating response."

# Define the Gradio interface
def create_interface() -> gr.Blocks():
    """
    Defines and returns the Gradio interface for the application.
    """
    with gr.Blocks() as demo:
        gr.Markdown("# All-in-One Scraper, Database, and RSS Feeder")

        with gr.Row():
            with gr.Column():
                # Scraping Controls
                storage_location = gr.Textbox(
                    value=DEFAULT_FILE_PATH, label="Storage Location"
                )
                urls = gr.Textbox(
                    label="URLs (comma separated)",
                    placeholder="https://example.com, https://anotherexample.com",
                )
                scrape_interval = gr.Slider(
                    minimum=1,
                    maximum=60,
                    value=5,
                    step=1,
                    label="Scrape Interval (minutes)",
                )
                content_type = gr.Radio(
                    choices=["text", "media", "both"],
                    value="text",
                    label="Content Type",
                )
                selector = gr.Textbox(
                    label="CSS Selector for Media (Optional)",
                    placeholder="e.g., img.main-image",
                )
                start_button = gr.Button("Start Scraping")
                stop_button = gr.Button("Stop Scraping")
                status_output = gr.Textbox(
                    label="Status Output", interactive=False, lines=2
                )

            with gr.Column():
                # Chat Interface
                chat_history = gr.Chatbot(label="Chat History", type='messages')
                with gr.Row():
                    message = gr.Textbox(label="Message", placeholder="Type your message here...")
                system_message = gr.Textbox(
                    value="You are a helpful assistant.", label="System message"
                )
                max_tokens = gr.Slider(
                    minimum=1,
                    maximum=2048,
                    value=512,
                    step=1,
                    label="Max new tokens",
                )
                temperature = gr.Slider(
                    minimum=0.1,
                    maximum=4.0,
                    value=0.7,
                    step=0.1,
                    label="Temperature",
                )
                top_p = gr.Slider(
                    minimum=0.1,
                    maximum=1.0,
                    value=0.95,
                    step=0.05,
                    label="Top-p (nucleus sampling)",
                )
                response_box = gr.Textbox(label="Response", interactive=False, lines=2)

        with gr.Row():
            with gr.Column():
                # CSV Display Controls
                selected_url_csv = gr.Textbox(
                    label="Select URL for CSV Content",
                    placeholder="https://example.com",
                )
                csv_button = gr.Button("Display CSV Content")
                csv_content_output = gr.Textbox(
                    label="CSV Content Output", interactive=False, lines=10
                )

            with gr.Column():
                # RSS Feed Generation Controls
                selected_url_rss = gr.Textbox(
                    label="Select URL for RSS Feed",
                    placeholder="https://example.com",
                )
                rss_button = gr.Button("Generate RSS Feed")
                rss_output = gr.Textbox(
                    label="RSS Feed Output", interactive=False, lines=20
                )

        # Historical Data View
        with gr.Row():
            with gr.Column():
                historical_view_url = gr.Textbox(
                    label="Select URL for Historical Data",
                    placeholder="https://example.com",
                )
                historical_button = gr.Button("View Historical Data")
                historical_output = gr.Dataframe(
                    headers=["ID", "URL", "Content Hash", "Change Detected"],
                    label="Historical Data",
                    interactive=False
                )

        # Connect buttons to their respective functions
        start_button.click(
            fn=start_scraping,
            inputs=[
                storage_location,
                urls,
                scrape_interval,
                content_type,
                selector,
            ],
            outputs=status_output,
        )

        stop_button.click(fn=stop_scraping, outputs=status_output)

        csv_button.click(
            fn=display_csv,
            inputs=[storage_location, selected_url_csv],
            outputs=csv_content_output,
        )

        rss_button.click(
            fn=generate_rss_feed,
            inputs=[storage_location, selected_url_rss],
            outputs=rss_output,
        )

        historical_button.click(
            fn=display_historical_data,
            inputs=[storage_location, historical_view_url],
            outputs=historical_output,
        )

        # Connect message submission to the chat interface
        def update_chat(message_input, history, system_msg, max_toks, temp, top_p_val):
            if not message_input.strip():
                return history, "Please enter a message."

            response = respond(
                message_input,
                history,
                system_msg,
                max_toks,
                temp,
                top_p_val,
            )
            history.append((message_input, response))
            return history, response

        message.submit(
            update_chat,
            inputs=[
                message,
                chat_history,
                system_message,
                max_tokens,
                temperature,
                top_p,
            ],
            outputs=[chat_history, response_box],
        )

    return demo
# Function to display historical data
def display_historical_data(storage_location: str, url: str):
    """
    Retrieves and displays historical scraping data for a given URL.
    """
    try:
        connection = get_db_connection()
        if connection:
            try:
                cursor = connection.cursor(dictionary=True)
                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC"
                cursor.execute(query, (url,))
                results = cursor.fetchall()

                if not results:
                    return pd.DataFrame()

                df = pd.DataFrame(results)
                cursor.close()
                connection.close()
                return df
            except mysql.connector.Error as err:
                logging.error(f"Error fetching historical data from database: {err}")
                # Fallback to CSV
        else:
            logging.info("No database connection. Fetching historical data from CSV.")

        # Fallback to CSV
        hostname = urlparse(url).hostname
        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
        if os.path.exists(csv_path):
            df = pd.read_csv(csv_path)
            return df
        else:
            return pd.DataFrame()
    except Exception as e:
        logging.error(f"Error fetching historical data for {url}: {e}")
        return pd.DataFrame()

    def load_model():
        """
        Loads the openLlama model and tokenizer once and returns the pipeline.
        """
    try:
        model_name = "openlm-research/open_llama_3b_v2"
        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, legacy=False)
        model = AutoModelForCausalLM.from_pretrained(model_name)

        # This should be inside the try block
        max_supported_length = 2048

        openllama_pipeline = pipeline(
            "text-generation",
            model=model,
            tokenizer=tokenizer,
            truncation=True,
            max_length=max_supported_length,
            temperature=0.7,
            top_p=0.95,
            device=0 if torch.cuda.is_available() else -1,
        )
        logging.info("Model loaded successfully.")
        return openllama_pipeline  # Return the pipeline
    except Exception as e:
        logging.error(f"Error loading google/flan-t5-xl model: {e}")
        return None

def load_model(model_name: str):
    """
    Loads the specified model and tokenizer.
    """
    try:
        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, legacy=False)
        model = AutoModelForCausalLM.from_pretrained(model_name)
        # This should be inside the try block
        max_supported_length = 2048  # Get this from the model config
        openllama_pipeline = pipeline(
            "text-generation",
            model=model,
            tokenizer=tokenizer,
            truncation=True,
            max_length=max_supported_length,
            temperature=0.7,
            top_p=0.95,
            device=0 if torch.cuda.is_available() else -1,
        )
        logging.info(f"{model_name} loaded successfully.")
        return openllama_pipeline
    except Exception as e:
        logging.error(f"Error loading {model_name} model: {e}")
        return None

# Automated Testing using unittest
class TestApp(unittest.TestCase):
    def test_parse_command_filter(self):
        command = "Filter apples, oranges in column Description"
        parsed_command = parse_command(command)
        self.assertEqual(parsed_command[0], "filter")
        self.assertListEqual(parsed_command[1]["words"], ["apples", "oranges"])
        self.assertEqual(parsed_command[1]["column"], "Description")

    def test_parse_command_sort(self):
        command = "Sort Price ascending"
        parsed_command = parse_command(command)
        self.assertEqual(parsed_command[0], "sort")
        self.assertEqual(parsed_command[1]["column"], "Price")
        self.assertEqual(parsed_command[1]["order"], "ascending")

    def test_parse_command_export(self):
        command = "Export to CSV as filtered_data.csv"
        parsed_command = parse_command(command)
        self.assertEqual(parsed_command[0], "export")
        self.assertEqual(parsed_command[1]["filename"], "filtered_data.csv")

    def test_parse_command_log(self):
        command = "Log action Filtered data for specific fruits"
        parsed_command = parse_command(command)
        self.assertEqual(parsed_command[0], "log")
        self.assertEqual(parsed_command[1]["action"], "Filtered data for specific fruits")

    def test_database_connection(self):
        connection = get_db_connection()
        # Connection may be None if not configured; adjust the test accordingly
        if connection:
            self.assertTrue(connection.is_connected())
            connection.close()
        else:
            self.assertIsNone(connection)

def main():
    # Initialize and run the application
    logging.info("Starting the application...")
    model = load_model()
    if model:
        logging.info("Application started successfully.")
        print("Main function executed")
        print("Creating interface...")
        demo = create_interface()
        print("Launching interface...")
        demo.launch(server_name="0.0.0.0", server_port=7860)
    else:
        logging.error("Failed to start the application.")

# Main execution
if __name__ == "__main__":
    # Initialize database
    initialize_database()

    # Create and launch Gradio interface
    demo = create_interface()
    demo.launch()

    # Run automated tests
    unittest.main(argv=[''], exit=False)