app.py

import os
import time
import hashlib
import logging
import datetime
import csv
from urllib.parse import urlparse
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
import requests
import feedparser
import gradio as gr

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# Define constants
DEFAULT_FILE_PATH = "scraped_data"
PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
HISTORY = []
CURRENT_TASK = None

# Define the list of URLs to monitor (you can add more URLs here)
URLS_TO_MONITOR = ["https://twitter.com/wlcscrdp", "https://www.facebook.com/aurorareddevils/", "https://www.facebook.com/brightpanthers/", "https://www.facebook.com/carrollcountychamberin/", "https://www.facebook.com/Culver.Cavs.MHS", "https://www.facebook.com/culver.elementary.school", "https://www.facebook.com/CulverCommunitySchools", "https://www.facebook.com/DillsboroBulldogs/", "https://www.facebook.com/ECMSTROJANS", "https://www.facebook.com/enjoywhitecountyIN/", "https://www.facebook.com/farmersvilleelementary", "https://www.facebook.com/groups/SDMSparents", "https://www.facebook.com/jghsart/", "https://www.facebook.com/jgmusicdept", "https://www.facebook.com/John-Glenn-Education-Foundation-208326199636364/", "https://www.facebook.com/John-Glenn-High-School-1102148953201006/", "https://www.facebook.com/John-Glenn-Theatre-Company-383638295064502/", "https://www.facebook.com/JohnGlennFalconsAthletics", "https://www.facebook.com/KIRPC-Head-Start-1485812354989001", "https://www.facebook.com/KIRPC1", "https://www.facebook.com/LHNEeagles", "https://www.facebook.com/LuceElementarySchool/", "https://www.facebook.com/marrselementary", "https://www.facebook.com/messhiners/", "https://www.facebook.com/monticellocitypool", "https://www.facebook.com/monticelloinwastewater/", "https://www.facebook.com/MooresHillBobcats/", "https://www.facebook.com/msdmv", "https://www.facebook.com/msdnorthposey", "https://www.facebook.com/MUTPL/", "https://www.facebook.com/MVJHS/", "https://www.facebook.com/mvshs", "https://www.facebook.com/njspjrsrhighschool?mibextid=b06tZ0", "https://www.facebook.com/NorthElementaryStars/", "https://www.facebook.com/NorthLibertyElementary/", "https://www.facebook.com/northposey/", "https://www.facebook.com/northposeyhs/", "https://www.facebook.com/NPJuniorHigh", "https://www.facebook.com/Prairie-Heights-Elementary-659322230934707/", "https://www.facebook.com/Prairie-Heights-High-School-2027713067459043/", "https://www.facebook.com/PrairieHeightsPanthers/", "https://www.facebook.com/profile.php?id=100057030237096", "https://www.facebook.com/profile.php?id=100057451179651", "https://www.facebook.com/profile.php?id=100063463513451", "https://www.facebook.com/profile.php?id=100063612319256", "https://www.facebook.com/profile.php?id=100064532596422", "https://www.facebook.com/profile.php?id=100067180226810", "https://www.facebook.com/profile.php?id=61563484312348", "https://www.facebook.com/PTOSWES/", "https://www.facebook.com/RandolphSouthern/", "https://www.facebook.com/RochesterMiddleSchool", "https://www.facebook.com/RochesterZebraNewTechHigh", "https://www.facebook.com/rockportelementarysouthspencer/", "https://www.facebook.com/satellitesathletics/", "https://www.facebook.com/seymourcommunityschools/", "https://www.facebook.com/SeymourHighSchool/", "https://www.facebook.com/SouthDearbornHighSchool/", "https://www.facebook.com/southarbornschools/", "https://www.facebook.com/SouthDearbornSquires/", "https://www.facebook.com/southspencerhighschool", "https://www.facebook.com/southspencermiddleschool/", "https://www.facebook.com/SouthSpencerSchools", "https://www.facebook.com/SouthTerracePanthers/", "https://www.facebook.com/sunmantigers/", "https://www.facebook.com/SWShelbySpartan/", "https://www.facebook.com/TallTimbersMarina", "https://www.facebook.com/WabashValleyESC/", "https://www.facebook.com/Walkerton-Elementary-School-283088605088622/", "https://www.facebook.com/westcentralcte/", "https://www.facebook.com/westelementary", "https://www.facebook.com/wlcscrdp", "https://www.instagram.com/mutpl/", "https://www.instagram.com/northposeyhsathletics", "https://www.instagram.com/rchsprincipalcook/", "https://www.instagram.com/southdearbornhighschool/", "https://www.instagram.com/southdearbornschools/", "https://www.instagram.com/westcentralcte/", "https://www.tiktok.com/@mutplteen"]

# Function to monitor URLs for changes
def monitor_urls(storage_location, urls, scrape_interval, content_type):
    global HISTORY
    previous_hashes = {url: "" for url in urls}  # Use a dictionary for better organization

    try:
        with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
            while True:
                for url in urls:
                    try:
                        driver.get(url)
                        time.sleep(2)  # Wait for the page to load
                        if content_type == "text":
                            current_content = driver.page_source
                        elif content_type == "media":
                            current_content = driver.find_elements(By.TAG_NAME, "img")
                        else:
                            current_content = driver.page_source
                        current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
                        if current_hash != previous_hashes[url]:
                            previous_hashes[url] = current_hash
                            date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                            HISTORY.append(f"Change detected at {url} on {date_time_str}")
                            with open(os.path.join(storage_location, f"{urlparse(url).hostname}_changes.csv"), "a", newline="") as csvfile:
                                csv_writer = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"])
                                csv_writer.writerow({"date": date_time_str.split()[0], "time": date_time_str.split()[1], "url": url, "change": "Content changed"})
                                logging.info(f"Change detected at {url} on {date_time_str}")
                    except (NoSuchElementException, Exception) as e:
                        logging.error(f"Error accessing {url}: {e}")
                time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
    except Exception as e:
        logging.error(f"Error starting ChromeDriver: {e}")

# Function to start scraping
def start_scraping(storage_location, urls, scrape_interval, content_type):
    global CURRENT_TASK, HISTORY

    CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
    HISTORY.append(f"Task started: {CURRENT_TASK}")

    for url in urls:
        # Create a folder for the URL
        hostname = urlparse(url).hostname
        folder_path = os.path.join(storage_location, hostname)
        os.makedirs(folder_path, exist_ok=True)

        # Log the initial observation
        try:
            with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
                driver.get(url)
                time.sleep(2)  # Wait for the page to load
                if content_type == "text":
                    initial_content = driver.page_source
                elif content_type == "media":
                    initial_content = driver.find_elements(By.TAG_NAME, "img")
                else:
                    initial_content = driver.page_source
                initial_hash = hashlib.md5(str(initial_content).encode('utf-8')).hexdigest()
                HISTORY.append(f"Initial observation at {url}: {initial_hash}")
                with open(os.path.join(folder_path, f"{hostname}_initial_observation.txt"), "w") as file:
                    file.write(f"Initial observation at {url}: {initial_hash}")
        except (NoSuchElementException, Exception) as e:
            HISTORY.append(f"Error accessing {url}: {e}")

    # Monitor the URLs
    monitor_urls(storage_location, urls, scrape_interval, content_type)

    return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."

# Function to display CSV content
def display_csv(url):
    hostname = urlparse(url).hostname
    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
    if os.path.exists(csv_path):
        with open(csv_path, "r") as file:
            return file.read()
    else:
        return "No data available."

# Define the chat response function using the Mistral model
def respond(message, history, system_message, max_tokens, temperature, top_p):
    API_URL = "https://api-inference.huggingface.co/models/mistralai/Mixtral-8x7B-Instruct-v0.1"
    headers = {"Authorization": f"Bearer {os.getenv('HF_API_TOKEN')}"}
    payload = {
        "inputs": f"User: {message}\nHistory: {history}\nSystem: {system_message}",
        "parameters": {"max_length": max_tokens, "temperature": temperature, "top_p": top_p},
    }
    response = requests.post(API_URL, headers=headers, json=payload)
    return response.json()[0]["generated_text"]

# Function to generate RSS feed for a given URL
def generate_rss_feed(url):
    hostname = urlparse(url).hostname
    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
    if os.path.exists(csv_path):
        with open(csv_path, "r") as file:
            reader = csv.DictReader(file)
            feed = feedparser.parse(f"rss.xml")  # Create a new feed object
            feed.feed.title = f"Changes for {hostname}"
            feed.feed.link = url
            feed.feed.description = "Recent changes detected on the website."
            feed.entries = []
            for row in reader:
                feed.entries.append({
                    "title": f"Change detected at {row['url']}",
                    "link": row['url'],
                    "description": f"Content changed on {row['date']} at {row['time']}",
                    "published": datetime.datetime.strptime(f"{row['date']} {row['time']}", "%Y-%m-%d %H:%M:%S").isoformat(),
                })
            return feed.entries
    else:
        return "No data available."

# Function to handle user input and generate response
def chat_interface(message, history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type):
    response = respond(message, history, system_message, max_tokens, temperature, top_p)
    history.append((message, response))
    return history, response

# Create Gradio interface
def create_interface():
    with gr.Blocks() as demo:
        with gr.Row():
            with gr.Column():
                message = gr.Textbox(label="Message")
                system_message = gr.Textbox(value="You are a helpful assistant.", label="System message")
                max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
                temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
                top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
                storage_location = gr.Textbox(value="scraped_data", label="Storage Location")
                urls = gr.Textbox(label="URLs (comma separated)")
                scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
                content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
                start_button = gr.Button("Start Scraping")
                csv_output = gr.Textbox(label="CSV Output", interactive=False)

            with gr.Column():
                chat_history = gr.Chatbot(label="Chat History")
                response_box = gr.Textbox(label="Response")

        start_button.click(start_scraping, inputs=[storage_location, urls, scrape_interval, content_type], outputs=csv_output)
        message.submit(chat_interface, inputs=[message, chat_history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type], outputs=[chat_history, response_box])

        # Add a button to display the RSS feed for a selected URL
        with gr.Row():
            selected_url = gr.Textbox(label="Select URL for RSS Feed")
            rss_button = gr.Button("Generate RSS Feed")
            rss_output = gr.Textbox(label="RSS Feed Output", interactive=False)

        rss_button.click(generate_rss_feed, inputs=[selected_url], outputs=rss_output)

    return demo


if __name__ == "__main__":
    interface = gr.Interface(fn=create_interface, title="Web Scraper and Chatbot")
    interface.launch()