Update app.py

app.py

@@ -1,139 +1,143 @@
-import time
-import csv
+import datetime
 import os
+import csv
+import time
 import hashlib
-import re
-import requests
-from bs4 import BeautifulSoup
+import logging
+import gradio as gr
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
-from inspector import Configuration, Inspector
-
-config = Configuration('5713ec1deb658fd2e6c069ce313ddaa34e2feee3')
-inspector = Inspector(config)
-inspector.start_transaction('my python script')
-
-def handle_input(input):
-    global CURRENT_INPUT
-    CURRENT_INPUT = input
-
-def handle_output(output):
-    global OUTPUT_HISTORY
-    OUTPUT_HISTORY.append((CURRENT_INPUT, output))
-
-def handle_system(output):
-    global SYSTEM_OUTPUT
-    SYSTEM_OUTPUT = output
-
-def handle_error(error):
-    global ERROR_HISTORY
-    ERROR_HISTORY.append((CURRENT_INPUT, error))
-
+from huggingface_hub import InferenceClient
+import random
+import yaml
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Define constants
+DATE_TIME_STR = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
+HISTORY = []
+CURRENT_TASK = None
+DEFAULT_FILE_PATH = "user/app/scraped_data/culver/culvers_changes.csv"
+
+# Ensure the directory exists
+os.makedirs(os.path.dirname(DEFAULT_FILE_PATH), exist_ok=True)
+
+# Function to monitor URLs for changes
+def monitor_urls(storage_location, urls, scrape_interval, content_type):
+    global HISTORY
+    previous_hashes = [""] * len(urls)
+
+    try:
+        with webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=Options()) as driver:
+            while True:
+                for i, url in enumerate(urls):
+                    try:
+                        driver.get(url)
+                        time.sleep(2)  # Wait for the page to load
+                        if content_type == "text":
+                            current_content = driver.page_source
+                        elif content_type == "media":
+                            current_content = driver.find_elements_by_tag_name("img")
+                        else:
+                            current_content = driver.page_source
+                        current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
+                        if current_hash != previous_hashes[i]:
+                            previous_hashes[i] = current_hash
+                            date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                            HISTORY.append(f"Change detected at {url} on {date_time_str}")
+                            with open(storage_location, "a", newline="") as csvfile:
+                                csv_writer = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"])
+                                csv_writer.writerow({"date": date_time_str.split()[0], "time": date_time_str.split()[1], "url": url, "change": "Content changed"})
+                                logging.info(f"Change detected at {url} on {date_time_str}")
+                    except Exception as e:
+                        logging.error(f"Error accessing {url}: {e}")
+                time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
+    except Exception as e:
+        logging.error(f"Error starting ChromeDriver: {e}")
+
+# Define main function to handle user input
+def handle_input(storage_location, urls, scrape_interval, content_type):
+    global CURRENT_TASK, HISTORY
+
+    CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
+    HISTORY.append(f"Task started: {CURRENT_TASK}")
+    monitor_urls(storage_location, urls, scrape_interval, content_type)
+    return TASK_PROMPT.format(task=CURRENT_TASK, history="\n".join(map(str, HISTORY)))
+
+# Load custom prompts
+try:
+    with open("custom_prompts.yaml", "r") as fp:
+        custom_prompts = yaml.safe_load(fp)
+except FileNotFoundError:
+    custom_prompts = {"WEB_DEV": "", "AI_SYSTEM_PROMPT": "", "PYTHON_CODE_DEV": "", "CODE_GENERATION": "", "CODE_INTERPRETATION": "", "CODE_TRANSLATION": "", "CODE_IMPLEMENTATION": ""}
+
+# Define agents
+AGENTS = ["WEB_DEV", "AI_SYSTEM_PROMPT", "PYTHON_CODE_DEV", "CODE_GENERATION", "CODE_INTERPRETATION", "CODE_TRANSLATION", "CODE_IMPLEMENTATION"]
+
+# Define the Mistral inference client
+client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+
+# Define the chat response function
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    return generate(message, history, system_message, max_tokens, temperature, top_p)
+
+# Function to start scraping
 def start_scraping(storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
     urls = [url for url in [url1, url2, url3, url4, url5, url6, url7, url8, url9, url10] if url]
-    handle_input(f"Start scraping {', '.join(urls)} every {scrape_interval} minutes.")
-    csv_file_path = f"{storage_location}/scraped_data.csv"
-    csv_fieldnames = ["date", "time", "url", "change"]
-
-    # Create the CSV file if it does not exist
-    if not os.path.exists(csv_file_path):
-        with open(csv_file_path, 'w', newline='') as csvfile:
-            csv_writer = csv.DictWriter(csvfile, fieldnames=csv_fieldnames)
-            csv_writer.writeheader()
-
-    while True:
-        # Check for scrape_interval
-        time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
-        # Scrape data
-        for url in urls:
-            # Initialize Chrome webdriver
-            options = Options()
-            options.headless = True
-            driver = webdriver.Chrome(executable_path='/path/to/chromedriver', options=options)
-            driver.maximize_window()
-            driver.set_window_size(1920, 1080)
-            driver.implicitly_wait(10)
-            driver.get(url)
-
-            # Wait for page to load
-            wait = WebDriverWait(driver, 10)
-            wait.until(EC.title_is('Culver Community Schools'))
-
-            # Scrape data
-            soup = BeautifulSoup(driver.page_source, 'html.parser')
-            content = None
-            if content_type == 'text':
-                content = soup.get_text()
-            elif content_type == 'media':
-                content = [img['src'] for img in soup.find_all('img')]
-            else:
-                raise Exception('Invalid content type')
-
-            # Calculate hash of the content
-            content_hash = hashlib.md5(str(content).encode('utf-8')).hexdigest()
-
-            # Check if the content has changed
-            with open(csv_file_path, 'r', newline='') as csvfile:
-                csv_reader = csv.DictReader(csvfile)
-                rows = list(csv_reader)
-                if rows:
-                    last_row = rows[-1]
-                    if last_row['url'] == url and last_row['change'] == content_hash:
-                        print(f"No changes detected on {url}")
-                        continue
-
-            # Save data to CSV file
-            with open(csv_file_path, 'a', newline='') as csvfile:
-                csv_writer = csv.DictWriter(csvfile, fieldnames=csv_fieldnames)
-                csv_writer.writerow({
-                    "date": datetime.datetime.now().strftime("%Y-%m-%d"),
-                    "time": datetime.datetime.now().strftime("%H:%M:%S"),
-                    "url": url,
-                    "change": content_hash
-                })
-
-            # Save data to file
-            with open(f"{storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape.{content_type}", 'w') as f:
-                if content_type == 'text':
-                    f.write(content)
-                elif content_type == 'media':
-                    for img in content:
-                        response = requests.get(img)
-                        with open(f"{storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape/{hashlib.md5(response.content).hexdigest()[:10]}.jpg", 'wb') as f:
-                            f.write(response.content)
-                else:
-                    raise Exception('Invalid content type')
-
-            handle_output(f"Scraped {url} and saved data to {csv_file_path}")
-            handle_output(f"Scraped {url} and saved data to {storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape.{content_type}")
-            inspector.end_transaction()
-
-    # Handle errors
-    for error in ERROR_HISTORY:
-        handle_error(error)
-
-    # Return scraping status
-    handle_output(f"Scraping {', '.join(urls)} every {scrape_interval} minutes.")
-
-def handle_system():
-    handle_output(f"System: {SYSTEM_OUTPUT}")
-
-def handle_ui(ui):
-    # Start scraping
-    urls = ['https://www.culver.org/', 'https://www.culver.org/about-us/', 'https://www.culver.org/academics/', 'https://www.culver.org/athletics/', 'https://www.culver.org/arts-and-humanities/', 'https://www.culver.org/fine-and-performing-arts/', 'https://www.culver.org/clubs/', 'https://www.culver.org/community-education/', 'https://www.culver.org/community-outreach/']
-    scrape_interval = 5  # Define the scrape interval
-    content_type = 'text'  # Define the content type
-    start_scraping('scrape_data', *urls, scrape_interval, content_type)
-
-if __name__ == '__main__':
-    # Read input
-    input = "Start scraping https://www.culver.org/ and save data to scrape_data directory."
-    # Call functions
-    handle_input(input)
-    handle_system()
-    # Run system
-    handle_ui()
+    handle_input(storage_location, urls, scrape_interval, content_type)
+    return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."
+
+# Function to display CSV content
+def display_csv(storage_location):
+    if os.path.exists(storage_location):
+        with open(storage_location, "r") as file:
+            return file.read()
+    else:
+        return "No data available."
+
+# Create Gradio interface
+def chat_interface(message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
+    global HISTORY
+    response = respond(message, HISTORY, system_message, max_tokens, temperature, top_p)
+    HISTORY.append((message, response))
+    return HISTORY, ""
+
+demo = gr.Blocks()
+
+with demo:
+    with gr.Row():
+        with gr.Column():
+            message = gr.Textbox(label="Message")
+            system_message = gr.Textbox(value="You are a friendly Chatbot.", label="System message")
+            max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
+            temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
+            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
+            storage_location = gr.Textbox(value=DEFAULT_FILE_PATH, label="Storage Location")
+            url1 = gr.Textbox(value="https://www.culver.k12.in/", label="URL 1")
+            url2 = gr.Textbox(value="https://www.facebook.com/CulverCommunitySchools", label="URL 2")
+            url3 = gr.Textbox(label="URL 3")
+            url4 = gr.Textbox(label="URL 4")
+            url5 = gr.Textbox(label="URL 5")
+            url6 = gr.Textbox(label="URL 6")
+            url7 = gr.Textbox(label="URL 7")
+            url8 = gr.Textbox(label="URL 8")
+            url9 = gr.Textbox(label="URL 9")
+            url10 = gr.Textbox(label="URL 10")
+            scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
+            content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
+            start_button = gr.Button("Start Scraping")
+            csv_output = gr.Textbox(label="CSV Output", interactive=False)
+        
+        with gr.Column():
+            chat_history = gr.Chatbot(label="Chat History")
+            response_box = gr.Textbox(label="Response")
+    
+    start_button.click(start_scraping, inputs=[storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=csv_output)
+    message.submit(chat_interface, inputs=[message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=[chat_history, response_box])
+
+if __name__ == "__main__":
+    demo.launch()