import datetime import os import csv import time import hashlib import logging import threading from pathlib import Path import gradio as gr from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from webdriver_manager.chrome import ChromeDriverManager from huggingface_hub import InferenceClient # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler("monitoring.log"), logging.StreamHandler() ] ) # Define constants PREFIX = "Task started at {date_time_str}. Purpose: {purpose}" TASK_PROMPT = "Current task: {task}. History:\n{history}" # Define purpose purpose = """ You monitor Culvers sites continuously, seeking changes since your last observation. Any new changes are logged and dumped into a CSV, stored in your log folder at user/app/scraped_data. """ # Initialize history and task variables history = [] current_task = None monitoring_thread = None stop_event = threading.Event() # Default file path using pathlib for cross-platform compatibility default_file_path = Path("user/app/scraped_data/culver/culvers_changes.csv") # Ensure the directory exists default_file_path.parent.mkdir(parents=True, exist_ok=True) def monitor_urls(storage_location, urls, scrape_interval, content_type, stop_event): """ Monitor the given URLs for changes and log them into a CSV file. Runs in a separate thread. """ global history previous_hashes = [""] * len(urls) storage_path = Path(storage_location) # Initialize CSV file: write header if file doesn't exist if not storage_path.exists(): with storage_path.open("w", newline='', encoding='utf-8') as csvfile: csv_toolkit = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"]) csv_toolkit.writeheader() options = Options() options.headless = True options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) try: while not stop_event.is_set(): for i, url in enumerate(urls): try: driver.get(url) time.sleep(2) # Wait for the page to load if content_type == "text": current_content = driver.page_source elif content_type == "media": images = driver.find_elements(By.TAG_NAME, "img") current_content = ''.join([img.get_attribute('src') for img in images]) elif content_type == "both": images = driver.find_elements(By.TAG_NAME, "img") current_content = driver.page_source + ''.join([img.get_attribute('src') for img in images]) else: current_content = driver.page_source current_hash = hashlib.md5(current_content.encode('utf-8')).hexdigest() if current_hash != previous_hashes[i]: previous_hashes[i] = current_hash date_time = datetime.datetime.now() date_time_str = date_time.strftime("%Y-%m-%d %H:%M:%S") history_entry = f"Change detected at {url} on {date_time_str}" history.append(history_entry) with storage_path.open("a", newline='', encoding='utf-8') as csvfile: csv_toolkit = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"]) csv_toolkit.writerow({ "date": date_time.strftime("%Y-%m-%d"), "time": date_time.strftime("%H:%M:%S"), "url": url, "change": "Content changed" }) logging.info(history_entry) except Exception as e: logging.error(f"Error accessing {url}: {e}") # Sleep in smaller intervals to allow quicker shutdown for _ in range(scrape_interval * 60): if stop_event.is_set(): break time.sleep(1) except Exception as e: logging.error(f"Unexpected error in monitoring thread: {e}") finally: driver.quit() logging.info("Monitoring thread has been stopped.") def start_monitoring(storage_location, url1, url2, scrape_interval, content_type): global current_task, monitoring_thread, stop_event, history if monitoring_thread and monitoring_thread.is_alive(): return "Monitoring is already running.", history history = [] current_task = f"Monitoring URLs: {url1}, {url2}" history.append(f"Task started: {current_task}") logging.info(current_task) stop_event.clear() urls = [url1, url2] monitoring_thread = threading.Thread( target=monitor_urls, args=(storage_location, urls, scrape_interval, content_type, stop_event), daemon=True ) monitoring_thread.start() return "Monitoring started.", history def stop_monitoring(): global current_task, monitoring_thread, stop_event, history if monitoring_thread and monitoring_thread.is_alive(): stop_event.set() monitoring_thread.join() history.append("Monitoring stopped by user.") logging.info("Monitoring stopped by user.") current_task = None return "Monitoring stopped.", history else: return "No monitoring task is currently running.", history # Define the chat response function client = InferenceClient("HuggingFaceH4/zephyr-7b-beta") def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, ): messages = [{"role": "system", "content": system_message}] for user_msg, assistant_msg in history: if user_msg: messages.append({"role": "user", "content": user_msg}) if assistant_msg: messages.append({"role": "assistant", "content": assistant_msg}) messages.append({"role": "user", "content": message}) response = "" try: for msg in client.chat_completion( messages, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p, ): token = msg.choices[0].delta.get("content", "") response += token yield response except Exception as e: logging.error(f"Error in chatbot response: {e}") yield "An error occurred while generating the response." # Create Gradio interface with gr.Blocks() as demo: gr.Markdown("# Culvers Site Monitor and Chatbot") gr.Markdown( "Monitor changes on Culvers' websites and log them into a CSV file. " "Also, chat with a friendly chatbot." ) with gr.Tab("Monitor"): with gr.Row(): storage_location = gr.Textbox( value=str(default_file_path), label="Storage Location", placeholder="Path to CSV file where changes will be logged" ) with gr.Row(): url1 = gr.Textbox( value="https://www.culver.k12.in.us/", label="URL 1", placeholder="First URL to monitor" ) url2 = gr.Textbox( value="https://www.facebook.com/CulverCommunitySchools", label="URL 2", placeholder="Second URL to monitor" ) with gr.Row(): scrape_interval = gr.Slider( minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)" ) content_type = gr.Radio( choices=["text", "media", "both"], value="text", label="Content Type" ) with gr.Row(): start_button = gr.Button("Start Monitoring") stop_button = gr.Button("Stop Monitoring") with gr.Row(): monitoring_status = gr.Textbox( value="No active monitoring.", label="Monitoring Status", interactive=False ) with gr.Row(): monitoring_history = gr.Textbox( value="", label="Monitoring History", lines=10, interactive=False ) with gr.Tab("Chatbot"): chatbot = gr.Chatbot(label="Chat with the Assistant") with gr.Row(): system_message = gr.Textbox( value="You are a friendly Chatbot.", label="System Message", visible=False ) with gr.Row(): user_input = gr.Textbox( label="You:", placeholder="Type your message here..." ) submit_button = gr.Button("Send") # Parameters max_tokens = gr.Slider( minimum=1, maximum=2048, value=512, step=1, label="Max new tokens" ) temperature = gr.Slider( minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature" ) top_p = gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)" ) # Define interactions def update_monitoring_history(message, history_text): return history_text + message + "\n" start_button.click( fn=start_monitoring, inputs=[storage_location, url1, url2, scrape_interval, content_type], outputs=[monitoring_status, monitoring_history], queue=False ) stop_button.click( fn=stop_monitoring, inputs=None, outputs=[monitoring_status, monitoring_history], queue=False ) def display_history(status, hist): return status, "\n".join(hist) # Update monitoring_status and monitoring_history periodically def refresh_monitoring(status, hist): return status, "\n".join(hist) user_input.submit( lambda msg, hist, sys, max_t, temp, tp: ( gr.update(value=hist + [(msg, "")]), respond(msg, hist, sys, max_t, temp, tp) ), inputs=[user_input, chatbot, system_message, max_tokens, temperature, top_p], outputs=[chatbot, chatbot] ) submit_button.click( lambda msg, hist, sys, max_t, temp, tp: ( gr.update(value=hist + [(msg, "")]), respond(msg, hist, sys, max_t, temp, tp) ), inputs=[user_input, chatbot, system_message, max_tokens, temperature, top_p], outputs=[chatbot, chatbot] ) if __name__ == "__main__": demo.launch()