Update app.py

app.py

@@ -1,53 +1,212 @@
+```python
 import os
-from agent import start_scraping, display_csv, generate_rss_feed, chat_interface
-import gradio as gr
+import time
+import hashlib
+import logging
+import datetime
+import csv
+from urllib.parse import urlparse
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+from selenium.common.exceptions import NoSuchElementException
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import pipeline
+import feedparser
+from bs4 import BeautifulSoup
+import threading
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Define constants
+DEFAULT_FILE_PATH = "scraped_data"
+PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
+HISTORY = []
+CURRENT_TASK = None
+STOP_THREADS = False
+
+# Define the list of URLs to monitor (you can add more URLs here)
+URLS_TO_MONITOR = ["https://twitter.com/wlcscrdp", "https://www.facebook.com/aurorareddevils/", "https://www.facebook.com/brightpanthers/", "https://www.facebook.com/carrollcountychamberin/", "https://www.facebook.com/Culver.Cavs.MHS", "https://www.facebook.com/culver.elementary.school", "https://www.facebook.com/CulverCommunitySchools", "https://www.facebook.com/DillsboroBulldogs/", "https://www.facebook.com/ECMSTROJANS", "https://www.facebook.com/enjoywhitecountyIN/", "https://www.facebook.com/farmersvilleelementary", "https://www.facebook.com/groups/SDMSparents", "https://www.facebook.com/jghsart/", "https://www.facebook.com/jgmusicdept", "https://www.facebook.com/John-Glenn-Education-Foundation-208326199636364/", "https://www.facebook.com/John-Glenn-High-School-1102148953201006/", "https://www.facebook.com/John-Glenn-Theatre-Company-383638295064502/", "https://www.facebook.com/JohnGlennFalconsAthletics", "https://www.facebook.com/KIRPC-Head-Start-1485812354989001", "https://www.facebook.com/KIRPC1", "https://www.facebook.com/LHNEeagles", "https://www.facebook.com/LuceElementarySchool/", "https://www.facebook.com/marrselementary", "https://www.facebook.com/messhiners/", "https://www.facebook.com/monticellocitypool", "https://www.facebook.com/monticelloinwastewater/", "https://www.facebook.com/MooresHillBobcats/", "https://www.facebook.com/msdmv", "https://www.facebook.com/msdnorthposey", "https://www.facebook.com/MUTPL/", "https://www.facebook.com/MVJHS/", "https://www.facebook.com/mvshs", "https://www.facebook.com/njspjrsrhighschool?mibextid=b06tZ0", "https://www.facebook.com/NorthElementaryStars/", "https://www.facebook.com/NorthLibertyElementary/", "https://www.facebook.com/northposey/", "https://www.facebook.com/northposeyhs/", "https://www.facebook.com/NPJuniorHigh", "https://www.facebook.com/Prairie-Heights-Elementary-659322230934707/", "https://www.facebook.com/Prairie-Heights-High-School-2027713067459043/", "https://www.facebook.com/PrairieHeightsPanthers/", "https://www.facebook.com/profile.php?id=100057030237096", "https://www.facebook.com/profile.php?id=100057451179651", "https://www.facebook.com/profile.php?id=100063463513451", "https://www.facebook.com/profile.php?id=100063612319256", "https://www.facebook.com/profile.php?id=100064532596422", "https://www.facebook.com/profile.php?id=100067180226810", "https://www.facebook.com/profile.php?id=61563484312348", "https://www.facebook.com/PTOSWES/", "https://www.facebook.com/RandolphSouthern/", "https://www.facebook.com/RochesterMiddleSchool", "https://www.facebook.com/RochesterZebraNewTechHigh", "https://www.facebook.com/rockportelementarysouthspencer/", "https://www.facebook.com/satellitesathletics/", "https://www.facebook.com/seymourcommunityschools/", "https://www.facebook.com/SeymourHighSchool/", "https://www.facebook.com/SouthDearbornHighSchool/", "https://www.facebook.com/southarbornschools/", "https://www.facebook.com/SouthDearbornSquires/", "https://www.facebook.com/southspencerhighschool", "https://www.facebook.com/southspencermiddleschool/", "https://www.facebook.com/SouthSpencerSchools", "https://www.facebook.com/SouthTerracePanthers/", "https://www.facebook.com/sunmantigers/", "https://www.facebook.com/SWShelbySpartan/", "https://www.facebook.com/TallTimbersMarina", "https://www.facebook.com/WabashValleyESC/", "https://www.facebook.com/Walkerton-Elementary-School-283088605088622/", "https://www.facebook.com/westcentralcte/", "https://www.facebook.com/westelementary", "https://www.facebook.com/wlcscrdp", "https://www.instagram.com/mutpl/", "https://www.instagram.com/northposeyhsathletics", "https://www.instagram.com/rchsprincipalcook/", "https://www.instagram.com/southdearbornhighschool/", "https://www.instagram.com/southdearbornschools/", "https://www.instagram.com/westcentralcte/", "https://www.tiktok.com/@mutplteen"]
+
+# Function to monitor URLs for changes
+def monitor_urls(storage_location, urls, scrape_interval, content_type):
+    global HISTORY, STOP_THREADS
+    previous_hashes = {url: "" for url in urls}  # Use a dictionary for better organization
+
+    try:
+        with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
+            while not STOP_THREADS:
+                for url in urls:
+                    try:
+                        driver.get(url)
+                        time.sleep(2)  # Wait for the page to load
+                        if content_type == "text":
+                            current_content = driver.page_source
+                        elif content_type == "media":
+                            current_content = driver.find_elements(By.TAG_NAME, "img")
+                        elif content_type == "both":
+                            current_content = driver.page_source + str(driver.find_elements(By.TAG_NAME, "img"))
+                        else:
+                            current_content = driver.page_source
+
+                        # Calculate hash based on selected content type
+                        current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
+
+                        if current_hash != previous_hashes[url]:
+                            previous_hashes[url] = current_hash
+                            date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                            HISTORY.append(f"Change detected at {url} on {date_time_str}")
+                            with open(os.path.join(storage_location, f"{urlparse(url).hostname}_changes.csv"), "a", newline="") as csvfile:
+                                csv_writer = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"])
+                                csv_writer.writerow({"date": date_time_str.split()[0], "time": date_time_str.split()[1], "url": url, "change": "Content changed"})
+                                logging.info(f"Change detected at {url} on {date_time_str}")
+
+                    except (NoSuchElementException, Exception) as e:
+                        logging.error(f"Error accessing {url}: {e}")
+                time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
+    except Exception as e:
+        logging.error(f"Error starting ChromeDriver: {e}")
+
+# Function to start scraping
+def start_scraping(storage_location, urls, scrape_interval, content_type):
+    global CURRENT_TASK, HISTORY, STOP_THREADS
+    STOP_THREADS = False  # Reset stop flag
+
+    CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
+    HISTORY.append(f"Task started: {CURRENT_TASK}")
+
+    for url in urls:
+        # Create a folder for the URL
+        hostname = urlparse(url).hostname
+        folder_path = os.path.join(storage_location, hostname)
+        os.makedirs(folder_path, exist_ok=True)
+
+        # Log the initial observation
+        try:
+            with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
+                driver.get(url)
+                time.sleep(2)  # Wait for the page to load
+                if content_type == "text":
+                    initial_content = driver.page_source
+                elif content_type == "media":
+                    initial_content = driver.find_elements(By.TAG_NAME, "img")
+                elif content_type == "both":
+                    initial_content = driver.page_source + str(driver.find_elements(By.TAG_NAME, "img"))
+                else:
+                    initial_content = driver.page_source
+                initial_hash = hashlib.md5(str(initial_content).encode('utf-8')).hexdigest()
+                HISTORY.append(f"Initial observation at {url}: {initial_hash}")
+                with open(os.path.join(folder_path, f"{hostname}_initial_observation.txt"), "w") as file:
+                    file.write(f"Initial observation at {url}: {initial_hash}")
+        except (NoSuchElementException, Exception) as e:
+            HISTORY.append(f"Error accessing {url}: {e}")
+
+    # Start a separate thread for monitoring
+    thread = threading.Thread(target=monitor_urls, args=(storage_location, urls, scrape_interval, content_type))
+    thread.start()
 
-def chatbot_interface(message, history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type, selector):
-    history, response = chat_interface(message, history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type, selector)
+    return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."
+
+# Function to display CSV content
+def display_csv(url):
+    hostname = urlparse(url).hostname
+    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
+    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
+    if os.path.exists(csv_path):
+        with open(csv_path, "r") as file:
+            return file.read()
+    else:
+        return "No data available."
+
+# Define the chat response function using the Mistral model
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    model = AutoModelForSeq2SeqLM.from_pretrained_model("mistralai/Mixtral-8x7B-Instruct-v0.1")
+    tokenizer = AutoTokenizer.from_pretrained_model("mistralai/Mixtral-8x7B-Instruct-v0.1")
+    pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+    response = pipe(f"User: {message}\nHistory: {history}\nSystem: {system_message}", max_length=max_tokens, temperature=temperature, top_p=top_p)[0]
+    return response
+
+# Function to generate RSS feed for a given URL
+def generate_rss_feed(url):
+    hostname = urlparse(url).hostname
+    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
+    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
+    if os.path.exists(csv_path):
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            feed = feedparser.parse(f"rss.xml")  # Create a new feed object
+            feed.feed.title = f"Changes for {hostname}"
+            feed.feed.link = url
+            feed.feed.description = "Recent changes detected on the website."
+            feed.entries = []
+            for row in reader:
+                feed.entries.append({
+                    "title": f"Change detected at {row['url']}",
+                    "link": row['url'],
+                    "description": f"Content changed on {row['date']} at {row['time']}",
+                    "published": datetime.datetime.strptime(f"{row['date']} {row['time']}", "%Y-%m-%d %H:%M:%S").isoformat(),
+                })
+            return feed.entries
+    else:
+        return "No data available."
+
+# Function to handle user input and generate response
+def chat_interface(message, history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type):
+    response = respond(message, history, system_message, max_tokens, temperature, top_p)
+    history.append((message, response))
     return history, response
 
-def generate_rss(storage_location, url):
-    feed_entries = generate_rss_feed(storage_location, url)
-    return feedparser.FeedParserDict(feed_entries)
-
-def main():
-    storage_location = "scraped_data"
-    urls = ["https://www.culvers.com/"]
-    scrape_interval = 5
-    content_type = "text"
-    selector = ""
-
-    chatbot_input = gr.inputs.Textbox(lines=5, label="Chatbot Input")
-    history_output = gr.outputs.Textbox(label="History")
-    response_output = gr.outputs.Textbox(label="Response")
-
-    csv_input = gr.inputs.Textbox(lines=1, label="CSV Input URL")
-    csv_output = gr.outputs.Textbox(label="CSV Output")
-
-    rss_input = gr.inputs.Textbox(lines=1, label="RSS Input URL")
-    rss_output = gr.outputs.HTML(label="RSS Output")
-
-    chatbot_interface = gr.Interface(
-        chatbot_interface,
-        [chatbot_input, history_output, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type, selector],
-        [history_output, response_output]
-    )
-
-    csv_interface = gr.Interface(
-        lambda url: display_csv(storage_location, url),
-        [csv_input],
-        [csv_output]
-    )
-
-    rss_interface = gr.Interface(
-        generate_rss,
-        [rss_input, storage_location],
-        [rss_output]
-    )
-
-    chatbot_interface.launch()
-    csv_interface.launch()
-    rss_interface.launch()
+# Gradio Interface
+import gradio as gr
+
+def create_interface():
+    with gr.Blocks() as demo:
+        with gr.Row():
+            with gr.Column():
+                message = gr.Textbox(label="Message")
+                system_message = gr.Textbox(value="You are a helpful assistant.", label="System message")
+                max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
+                temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
+                top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
+                storage_location = gr.Textbox(value="scraped_data", label="Storage Location")
+                urls = gr.Textbox(label="URLs (comma separated)")
+                scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
+                content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
+                start_button = gr.Button("Start Scraping")
+                stop_button = gr.Button("Stop Scraping")
+                csv_output = gr.Textbox(label="CSV Output", interactive=False)
+
+            with gr.Column():
+                chat_history = gr.Chatbot(label="Chat History")
+                response_box = gr.Textbox(label="Response")
+
+        start_button.click(start_scraping, inputs=[storage_location, urls, scrape_interval, content_type], outputs=csv_output)
+        stop_button.click(lambda: (STOP_THREADS, "Scraping stopped."), outputs=[stop_button, csv_output])
+        message.submit(chat_interface, inputs=[message, chat_history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type], outputs=[chat_history, response_box])
+
+        # Add a button to display the CSV content for a selected URL
+        with gr.Row():
+            selected_url = gr.Textbox(label="Select URL for CSV Content")
+            csv_button = gr.Button("Display CSV Content")
+            csv_output = gr.Textbox(label="CSV Content Output", interactive=False)
+
+        csv_button.click(display_csv, inputs=[selected_url], outputs=csv_output)
+
+        # Add a button to display the RSS feed for a selected URL
+        with gr.Row():
+            selected_url = gr.Textbox(label="Select URL for RSS Feed")
+            rss_button = gr.Button("Generate RSS Feed")
+            rss_output = gr.Textbox(label="RSS Feed Output", interactive=False)
+
+        rss_button.click(generate_rss_feed, inputs=[selected_url], outputs=rss_output)
+
+    return demo
 
 if __name__ == "__main__":
-    main()
+    demo = create_interface()
+    demo.launch()