import gradio as gr import pandas as pd import sqlite3 from feedgen.feed import FeedGenerator import datetime import os import logging import sys import csv import traceback sys.path.append('/home/user') from app.background_tasks import start_background_monitoring, create_database # Set up absolute paths BASE_DIR = '/home/user/app/scraped_data/culver' LOG_FILE = os.path.join(BASE_DIR, 'main.log') CSV_FILE = os.path.join(BASE_DIR, 'culvers_changes.csv') DB_FILE = os.path.join(BASE_DIR, 'culvers_changes.db') XML_FILE = os.path.join(BASE_DIR, 'culvers_changes.xml') # Ensure the directory exists try: os.makedirs(BASE_DIR, exist_ok=True) print(f"Directory created or already exists: {BASE_DIR}") except Exception as e: print(f"Error creating directory: {e}") traceback.print_exc() # Configure logging try: logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') print(f"Logging configured. Log file: {LOG_FILE}") except Exception as e: print(f"Error configuring logging: {e}") traceback.print_exc() # Write directly to log file try: with open(LOG_FILE, 'w') as log_file: log_file.write(f"Log file created at {datetime.datetime.now()}\n") print(f"Log file created: {LOG_FILE}") except Exception as e: print(f"Error writing to log file: {e}") traceback.print_exc() # Write directly to CSV file try: with open(CSV_FILE, 'w', newline='') as csv_file: writer = csv.writer(csv_file) writer.writerow(['date', 'time', 'url', 'change']) writer.writerow([datetime.datetime.now().strftime("%Y-%m-%d"), datetime.datetime.now().strftime("%H:%M:%S"), 'Initial', 'CSV file created']) print(f"CSV file created: {CSV_FILE}") except Exception as e: print(f"Error writing to CSV file: {e}") traceback.print_exc() # Start background monitoring urls = ["https://www.culver.k12.in.us/", "https://www.facebook.com/CulverCommunitySchools"] try: start_background_monitoring(CSV_FILE, urls, 1, "text") # Changed interval to 1 minute for testing print("Background monitoring started") except Exception as e: print(f"Error starting background monitoring: {e}") traceback.print_exc() logging.info("Background monitoring initiated from main.py") def view_scraped_data(): try: create_database() # Ensure the database and table exist conn = sqlite3.connect(DB_FILE) df = pd.read_sql_query("SELECT * FROM changes ORDER BY date DESC, time DESC LIMIT 50", conn) conn.close() return df except Exception as e: print(f"Error viewing scraped data: {e}") traceback.print_exc() return pd.DataFrame() def view_rss_feed(): try: with open(XML_FILE, 'r') as file: return file.read() except FileNotFoundError: return "RSS feed not generated yet." except Exception as e: print(f"Error viewing RSS feed: {e}") traceback.print_exc() return "Error viewing RSS feed" def generate_rss_feed(): try: create_database() # Ensure the database and table exist fg = FeedGenerator() fg.title('Culvers Site Changes') fg.link(href='http://example.com', rel='alternate') fg.description('Recent changes detected on Culvers websites') conn = sqlite3.connect(DB_FILE) c = conn.cursor() c.execute("SELECT * FROM changes ORDER BY date DESC, time DESC LIMIT 20") changes = c.fetchall() for change in changes: fe = fg.add_entry() fe.id(str(change[0])) fe.title(f'Change detected at {change[3]}') fe.link(href=change[3]) fe.description(change[4]) fe.pubDate(datetime.datetime.strptime(f"{change[1]} {change[2]}", "%Y-%m-%d %H:%M:%S")) conn.close() fg.rss_file(XML_FILE) return "RSS feed generated successfully." except Exception as e: print(f"Error generating RSS feed: {e}") traceback.print_exc() return "Error generating RSS feed" def create_viewer(): with gr.Blocks() as demo: gr.Markdown("# Culvers Site Monitor and Viewer") with gr.Tab("Monitor Status"): gr.Markdown("Continuous monitoring is active for the following URLs:") for url in urls: gr.Markdown(f"- {url}") gr.Markdown(f"Monitoring interval: 1 minute") gr.Markdown(f"Data is being stored in: {CSV_FILE}") with gr.Tab("View Scraped Data"): gr.DataFrame(view_scraped_data, label="Recent Changes") with gr.Tab("View RSS Feed"): gr.TextArea(view_rss_feed, label="RSS Feed Content") gr.Button("Generate RSS Feed").click(generate_rss_feed, outputs=gr.TextArea(label="Generation Status")) return demo if __name__ == "__main__": try: # Create the database and table before launching the viewer create_database() print("Database created") # Create and launch the viewer viewer = create_viewer() print("Viewer created") viewer.launch() print("Viewer launched") logging.info("Web-based viewer created and launched with continuous monitoring.") except Exception as e: print(f"Error in main execution: {e}") traceback.print_exc() print("Main application file updated with error handling, console logging, and all necessary functions.")