Update app.py

app.py

@@ -1,150 +1,114 @@
-import datetime
-import os
-import csv
 import time
+import csv
+import os
 import hashlib
-import logging
-import gradio as gr
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
+import re
+import requests
+from bs4 import BeautifulSoup
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
 from webdriver_manager.chrome import ChromeDriverManager
-from huggingface_hub import InferenceClient
-import random
-import yaml
-
-# Configure logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-# Define constants
-DATE_TIME_STR = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
-HISTORY = []
-CURRENT_TASK = None
-DEFAULT_FILE_PATH = "user/app/scraped_data/culver/culvers_changes.csv"
-
-# Ensure the directory exists
-os.makedirs(os.path.dirname(DEFAULT_FILE_PATH), exist_ok=True)
-
-# Function to monitor URLs for changes
-def monitor_urls(storage_location, urls, scrape_interval, content_type):
-    global HISTORY
-    previous_hashes = [""] * len(urls)
-
-    try:
-        with webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=Options()) as driver:
-            while True:
-                for i, url in enumerate(urls):
-                    try:
-                        driver.get(url)
-                        time.sleep(2)  # Wait for the page to load
-                        if content_type == "text":
-                            current_content = driver.page_source
-                        elif content_type == "media":
-                            current_content = driver.find_elements_by_tag_name("img")
-                        else:
-                            current_content = driver.page_source
-                        current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
-                        if current_hash != previous_hashes[i]:
-                            previous_hashes[i] = current_hash
-                            date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                            HISTORY.append(f"Change detected at {url} on {date_time_str}")
-                            with open(storage_location, "a", newline="") as csvfile:
-                                csv_writer = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"])
-                                csv_writer.writerow({"date": date_time_str.split()[0], "time": date_time_str.split()[1], "url": url, "change": "Content changed"})
-                                logging.info(f"Change detected at {url} on {date_time_str}")
-                    except Exception as e:
-                        logging.error(f"Error accessing {url}: {e}")
-                time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
-    except Exception as e:
-        logging.error(f"Error starting ChromeDriver: {e}")
-
-# Define main function to handle user input
-def handle_input(storage_location, urls, scrape_interval, content_type):
-    global CURRENT_TASK, HISTORY
-
-    CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
-    HISTORY.append(f"Task started: {CURRENT_TASK}")
-    monitor_urls(storage_location, urls, scrape_interval, content_type)
-    return TASK_PROMPT.format(task=CURRENT_TASK, history="\n".join(map(str, HISTORY)))
-
-# Load custom prompts
-try:
-    with open("custom_prompts.yaml", "r") as fp:
-        custom_prompts = yaml.safe_load(fp)
-except FileNotFoundError:
-    custom_prompts = {"WEB_DEV": "", "AI_SYSTEM_PROMPT": "", "PYTHON_CODE_DEV": "", "CODE_GENERATION": "", "CODE_INTERPRETATION": "", "CODE_TRANSLATION": "", "CODE_IMPLEMENTATION": ""}
-
-# Define agents
-AGENTS = ["WEB_DEV", "AI_SYSTEM_PROMPT", "PYTHON_CODE_DEV", "CODE_GENERATION", "CODE_INTERPRETATION", "CODE_TRANSLATION", "CODE_IMPLEMENTATION"]
-
-# Define the Mistral inference client
-client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
-
-# Define the chat response function
-def respond(message, history, system_message, max_tokens, temperature, top_p):
-    return generate(message, history, system_message, max_tokens, temperature, top_p)
+from inspector import Configuration, Inspector
+
+config = Configuration('5713ec1deb658fd2e6c069ce313ddaa34e2feee3')
+inspector = Inspector(config)
+inspector.start_transaction('my python script')
+
+def handle_input(input):
+    global CURRENT_INPUT
+    CURRENT_INPUT = input
+
+def handle_output(output):
+    global OUTPUT_HISTORY
+    OUTPUT_HISTORY.append((CURRENT_INPUT, output))
+
+def handle_system(output):
+    global SYSTEM_OUTPUT
+    SYSTEM_OUTPUT = output
+
+def handle_error(error):
+    global ERROR_HISTORY
+    ERROR_HISTORY.append((CURRENT_INPUT, error))
 
 def start_scraping(storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
     urls = [url for url in [url1, url2, url3, url4, url5, url6, url7, url8, url9, url10] if url]
-    handle_input(storage_location, urls, scrape_interval, content_type)
-    # Start transaction
-    inspector.start_transaction('start_scraping')
-    # Scrape data
+    handle_input(f"Start scraping {', '.join(urls)} every {scrape_interval} minutes.")
     while True:
         # Check for scrape_interval
         time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
+        # Scrape data
+        for url in urls:
+            # Initialize Chrome webdriver
+            options = Options()
+            options.headless = True
+            driver = webdriver.Chrome(executable_path='/path/to/chromedriver', options=options)
+            driver.maximize_window()
+            driver.set_window_size(1920, 1080)
+            driver.implicitly_wait(10)
+            driver.get(url)
+
+            # Wait for page to load
+            wait = WebDriverWait(driver, 10)
+            wait.until(EC.title_is('Culver Community Schools'))
+
+            # Scrape data
+            soup = BeautifulSoup(driver.page_source, 'html.parser')
+            content = None
+            if content_type == 'text':
+                content = soup.get_text()
+            elif content_type == 'media':
+                content = [img['src'] for img in soup.find_all('img')]
+            else:
+                raise Exception('Invalid content type')
+
+            # Save data
+            with open(f"{storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape.{content_type}", 'w') as f:
+                if content_type == 'text':
+                    f.write(content)
+                elif content_type == 'media':
+                    for img in content:
+                        response = requests.get(img)
+                        with open(f"{storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape/{hashlib.md5(response.content).hexdigest()[:10]}.jpg", 'wb') as f:
+                            f.write(response.content)
+                else:
+                    raise Exception('Invalid content type')
+
+            # End transaction
+            inspector.end_transaction()
+            handle_output(f"Scraped {url} and saved data to {storage_location}/{url.split('/')[-2]}/{url.split('/')[-1]}_scrape.{content_type}")
+
     # End transaction
     inspector.end_transaction()
-    return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."
-
-# Function to display CSV content
-def display_csv(storage_location):
-    if os.path.exists(storage_location):
-        with open(storage_location, "r") as file:
-            return file.read()
-    else:
-        return "No data available."
-
-# Create Gradio interface
-def chat_interface(message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
-    global HISTORY
-    response = respond(message, HISTORY, system_message, max_tokens, temperature, top_p)
-    HISTORY.append((message, response))
-    return HISTORY, ""
-
-demo = gr.Blocks()
-
-with demo:
-    with gr.Row():
-        with gr.Column():
-            message = gr.Textbox(label="Message")
-            system_message = gr.Textbox(value="You are a friendly Chatbot.", label="System message")
-            max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
-            temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
-            storage_location = gr.Textbox(value=DEFAULT_FILE_PATH, label="Storage Location")
-            url1 = gr.Textbox(value="https://www.culver.k12.in/", label="URL 1")
-            url2 = gr.Textbox(value="https://www.facebook.com/CulverCommunitySchools", label="URL 2")
-            url3 = gr.Textbox(label="URL 3")
-            url4 = gr.Textbox(label="URL 4")
-            url5 = gr.Textbox(label="URL 5")
-            url6 = gr.Textbox(label="URL 6")
-            url7 = gr.Textbox(label="URL 7")
-            url8 = gr.Textbox(label="URL 8")
-            url9 = gr.Textbox(label="URL 9")
-            url10 = gr.Textbox(label="URL 10")
-            scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
-            content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
-            start_button = gr.Button("Start Scraping")
-            csv_output = gr.Textbox(label="CSV Output", interactive=False)
-        
-        with gr.Column():
-            chat_history = gr.Chatbot(label="Chat History")
-            response_box = gr.Textbox(label="Response")
-    
-    start_button.click(start_scraping, inputs=[storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=csv_output)
-    message.submit(chat_interface, inputs=[message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=[chat_history, response_box])
-
-if __name__ == "__main__":
-    demo.launch()
+
+    # Handle errors
+    for error in ERROR_HISTORY:
+        handle_error(error)
+
+    # Return scraping status
+    handle_output(f"Scraping {', '.join(urls)} every {scrape_interval} minutes.")
+
+def handle_system():
+    handle_output(f"System: {SYSTEM_OUTPUT}")
+
+def handle_ui(ui):
+    # Start scraping
+    start_scraping('scrape_data', 'https://www.culver.org/', 'https://www.culver.org/about-us/', 'https://www.culver.org/academics/', 'https://www.culver.org/athletics/', 'https://www.culver.org/arts-and-humanities/', 'https://www.culver.org/fine-and-performing-arts/', 'https://www.culver.org/clubs/', 'https://www.culver.org/community-education/', 'https://www.culver.org/community-outreach/')
+
+    # Handle errors
+    for error in ERROR_HISTORY:
+        handle_error(error)
+
+    # Return scraping status
+    handle_output(f"Scraping {', '.join(urls)} every {scrape_interval} minutes.")
+
+if __name__ == '__main__':
+    # Read input
+    input = "Start scraping https://www.culver.org/ and save data to scrape_data directory."
+    # Call functions
+    handle_input(input)
+    handle_system()
+    # Run system
+    handle_ui()