App.py

app.py

@@ -0,0 +1,505 @@
+import requests
+import subprocess
+import time
+import shutil
+import sys
+import gradio as gr
+from datetime import datetime, timedelta
+import json
+
+# --- Setup Ollama and Model ---
+def setup_ollama_once():
+    if shutil.which("ollama") is None:
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "--quiet",
+             "langchain", "langchain_community", "gradio", "requests"],
+            check=True
+        )
+        subprocess.run("curl -fsSL https://ollama.com/install.sh | sh", shell=True, check=True)
+        subprocess.Popen("ollama serve", shell=True)
+        time.sleep(8)
+        subprocess.run(["ollama", "pull", "llama3.1"], check=True)
+
+setup_ollama_once()
+
+from langchain_community.llms import Ollama
+_llama_client = Ollama(model="llama3.1")
+
+# --- API Keys ---
+GRIDSTATUS_API_KEY = "e9f1146bc3d242d19f7d64cf00f9fdd3"
+WEATHERAPI_KEY = "6325087c017744b0b0d13950252307"
+EIA_API_KEY = "l0LsSOevbx1XqUMSEdEP7qVwDQXoeC3bFw8LPdGZ"
+SERPAPI_KEY = "83fc2175c280b9187692d652ee4bb8bbcdfc652b0b8ea8539d7b494ac08280f3"
+
+# --- SerpAPI Integration for Situational Awareness ---
+def search_grid_emergencies():
+    """
+    Search for current emergencies that could impact grid operations
+    """
+    emergency_queries = [
+        "New York power grid emergency outage blackout today",
+        "Northeast wildfire power line damage 2025",
+        "New York electrical grid failure infrastructure",
+        "NYISO emergency alert grid operations"
+    ]
+
+    all_results = []
+
+    for query in emergency_queries:
+        try:
+            url = "https://serpapi.com/search"
+            params = {
+                "engine": "google",
+                "q": query,
+                "api_key": SERPAPI_KEY,
+                "num": 3,  # Limit results per query
+                "tbm": "nws",  # News search
+                "tbs": "qdr:d"  # Last 24 hours
+            }
+
+            response = requests.get(url, params=params, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+
+            if "news_results" in data:
+                for result in data["news_results"]:
+                    all_results.append({
+                        "type": "emergency",
+                        "title": result.get("title", ""),
+                        "snippet": result.get("snippet", ""),
+                        "source": result.get("source", ""),
+                        "date": result.get("date", "")
+                    })
+
+            time.sleep(1)  # Rate limiting
+
+        except Exception as e:
+            print(f"[ERROR] Emergency search failed for '{query}': {e}")
+            continue
+
+    return all_results
+
+def search_high_demand_events():
+    """
+    Search for events that could increase electricity demand
+    """
+    # Get today's date for relevant searches
+    today = datetime.now()
+    tomorrow = today + timedelta(days=1)
+    date_str = today.strftime("%Y-%m-%d")
+
+    demand_queries = [
+        f"New York City major events concerts sports {date_str}",
+        f"NYC heat wave extreme weather {today.strftime('%B %Y')}",
+        f"Super Bowl NYC watch parties high electricity demand",
+        f"New York massive concert Madison Square Garden today",
+        f"NYC extreme cold weather heating demand {today.strftime('%B %Y')}"
+    ]
+
+    all_results = []
+
+    for query in demand_queries:
+        try:
+            url = "https://serpapi.com/search"
+            params = {
+                "engine": "google",
+                "q": query,
+                "api_key": SERPAPI_KEY,
+                "num": 3,
+                "tbm": "nws",
+                "tbs": "qdr:w"  # Last week
+            }
+
+            response = requests.get(url, params=params, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+
+            if "news_results" in data:
+                for result in data["news_results"]:
+                    all_results.append({
+                        "type": "high_demand",
+                        "title": result.get("title", ""),
+                        "snippet": result.get("snippet", ""),
+                        "source": result.get("source", ""),
+                        "date": result.get("date", "")
+                    })
+
+            time.sleep(1)  # Rate limiting
+
+        except Exception as e:
+            print(f"[ERROR] High demand search failed for '{query}': {e}")
+            continue
+
+    return all_results
+
+def get_situational_awareness():
+    """
+    Combine emergency and high-demand event searches for comprehensive situational awareness
+    """
+    print("[DEBUG] Fetching situational awareness data...")
+
+    emergencies = search_grid_emergencies()
+    high_demand_events = search_high_demand_events()
+
+    # Combine and deduplicate results
+    all_events = emergencies + high_demand_events
+
+    # Remove duplicates based on title similarity
+    unique_events = []
+    seen_titles = set()
+
+    for event in all_events:
+        title_words = set(event["title"].lower().split())
+        is_duplicate = False
+
+        for seen_title in seen_titles:
+            seen_words = set(seen_title.lower().split())
+            # If more than 50% of words overlap, consider it duplicate
+            overlap = len(title_words.intersection(seen_words))
+            if overlap > len(title_words) * 0.5:
+                is_duplicate = True
+                break
+
+        if not is_duplicate:
+            unique_events.append(event)
+            seen_titles.add(event["title"])
+
+    print(f"[DEBUG] Found {len(unique_events)} unique situational awareness events")
+    return unique_events
+
+def format_situational_summary(events):
+    """
+    Format situational awareness events for LLM consumption
+    """
+    if not events:
+        return "No immediate emergencies or high-demand events detected."
+
+    emergency_events = [e for e in events if e["type"] == "emergency"]
+    demand_events = [e for e in events if e["type"] == "high_demand"]
+
+    summary = []
+
+    if emergency_events:
+        summary.append("CRITICAL ALERTS:")
+        for event in emergency_events[:3]:  # Limit to top 3
+            summary.append(f"- {event['title']}")
+            if event['snippet']:
+                summary.append(f"  {event['snippet'][:150]}...")
+
+    if demand_events:
+        summary.append("\nHIGH DEMAND EVENTS:")
+        for event in demand_events[:3]:  # Limit to top 3
+            summary.append(f"- {event['title']}")
+            if event['snippet']:
+                summary.append(f"  {event['snippet'][:150]}...")
+
+    return "\n".join(summary)
+
+# --- Fetch Load Data using EIA API (Fixed) ---
+def fetch_eia_load_direct():
+    """
+    Use EIA API to get NYISO load data directly
+    This replaces the broken GridStatus API call
+    """
+    try:
+        url = "https://api.eia.gov/v2/electricity/rto/region-data/data/"
+        params = {
+            "api_key": EIA_API_KEY,
+            "facets[respondent]": "NYIS",  # NYISO code in EIA
+            "facets[type-name]": "D",      # Demand/Load
+            "data[0]": "value",
+            "sort[0][column]": "period",
+            "sort[0][direction]": "desc",
+            "offset": 0,
+            "length": 1  # Just get the latest
+        }
+
+        response = requests.get(url, params=params, timeout=10)
+        response.raise_for_status()
+        data = response.json()
+
+        if not data["response"]["data"]:
+            raise ValueError("No load data returned from EIA")
+
+        latest_record = data["response"]["data"][0]
+        current_load = latest_record["value"]
+
+        print(f"[DEBUG] EIA Load for NYISO: {current_load} MW")
+        return float(current_load)
+
+    except Exception as e:
+        print(f"[ERROR] EIA load fetch failed: {e}")
+        return f"Error fetching EIA load data: {e}"
+
+# --- Alternative: Try NYISO Direct API as Backup ---
+def fetch_nyiso_direct_load():
+    """
+    Backup method: Use NYISO's direct API for load data
+    """
+    try:
+        from datetime import datetime
+        today = datetime.now().strftime("%Y%m%d")
+        url = f"http://mis.nyiso.com/public/csv/pal/{today}pal.csv"
+
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+
+        # Parse CSV data to get latest load
+        lines = response.text.strip().split('\n')
+        if len(lines) < 2:
+            raise ValueError("Invalid CSV response from NYISO")
+
+        # Get the last row (most recent data)
+        latest_data = lines[-1].split(',')
+        # NYISO load is typically in the second column after timestamp
+        current_load = float(latest_data[1])
+
+        print(f"[DEBUG] NYISO Direct Load: {current_load} MW")
+        return current_load
+
+    except Exception as e:
+        print(f"[ERROR] NYISO direct fetch failed: {e}")
+        return f"Error fetching NYISO direct data: {e}"
+
+# --- Main Load Fetch Function with Fallbacks ---
+def fetch_load_with_fallbacks():
+    """
+    Try multiple sources for load data with fallbacks
+    """
+    # Try EIA first
+    load = fetch_eia_load_direct()
+    if isinstance(load, float):
+        return load
+
+    print("[DEBUG] EIA failed, trying NYISO direct...")
+    # Try NYISO direct as backup
+    load = fetch_nyiso_direct_load()
+    if isinstance(load, float):
+        return load
+
+    print("[DEBUG] All load sources failed, using estimated value")
+    # If all else fails, return a reasonable estimate based on typical NYISO load
+    return 18000.0  # MW - typical NYISO load
+
+# --- Fetch Weather ---
+def fetch_weather_forecast(city="New York"):
+    url = f"http://api.weatherapi.com/v1/current.json?key={WEATHERAPI_KEY}&q={city}&aqi=no"
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        temp = data["current"]["temp_f"]
+        condition = data["current"]["condition"]["text"]
+        return temp, condition
+    except Exception as e:
+        return None, f"Weather fetch error: {e}"
+
+# --- Fetch NY Generator Profiles (from EIA) ---
+def get_ny_generator_profiles():
+    url = "https://api.eia.gov/v2/electricity/generator/data/"
+    params = {
+        "api_key": EIA_API_KEY,
+        "facets[state]": "NY",
+        "facets[operational_status_code]": "OP",
+        "data[0]": "nameplate_capacity",
+        "data[1]": "energy_source",
+        "data[2]": "prime_mover",
+        "data[3]": "plant_name",
+        "offset": 0,
+        "length": 100
+    }
+
+    try:
+        response = requests.get(url, params=params, timeout=10)
+        response.raise_for_status()
+        raw = response.json()
+        plants = raw["response"]["data"]
+        print(f"[DEBUG] Retrieved {len(plants)} plants")
+
+        profiles = []
+        for p in plants:
+            capacity = p.get("nameplate_capacity", 0)
+            if capacity < 10:
+                continue
+            fuel = p.get("energy_source", "UNK")
+            mover = p.get("prime_mover", "UNK")
+            plant = p.get("plant_name", "Unknown")
+
+            if fuel in ["WND", "SUN"]:
+                category = "renewable"
+            elif fuel == "NUC":
+                category = "baseload"
+            elif mover in ["CT", "GT"]:
+                category = "peaker"
+            elif mover in ["CC", "ST"]:
+                category = "midload"
+            else:
+                category = "misc"
+
+            profiles.append({
+                "name": plant,
+                "fuel": fuel,
+                "mover": mover,
+                "capacity": capacity,
+                "category": category
+            })
+
+        return profiles
+    except Exception as e:
+        print(f"[DEBUG] Generator profile fetch error: {e}")
+        return []
+
+# --- Core Logic: Enhanced Grid AI with Situational Awareness ---
+def real_time_decision_with_situational_awareness():
+    # Fetch all data sources
+    load = fetch_load_with_fallbacks()
+    temp, weather_desc = fetch_weather_forecast("New York")
+
+    # NEW: Get situational awareness data
+    situational_events = get_situational_awareness()
+    situational_summary = format_situational_summary(situational_events)
+
+    # Handle weather fetch failure
+    if temp is None:
+        temp = 70  # Default temperature if weather fetch fails
+        weather_desc = "Weather data unavailable"
+        print(f"[DEBUG] Weather fetch failed, using defaults: {temp}°F, {weather_desc}")
+
+    plant_data = get_ny_generator_profiles()
+    if not plant_data:
+        print("[DEBUG] Generator profiles unavailable, using generic recommendations")
+        plant_summary = "Generator profile data temporarily unavailable"
+    else:
+        plant_summary = "\n".join([
+            f"- {p['name']} ({p['category']}, {p['fuel']}, {p['capacity']} MW)"
+            for p in plant_data[:10]
+        ])
+
+    # Enhanced prompt with situational awareness
+    prompt = f"""
+You are a senior electric grid operator managing the New York Independent System Operator (NYISO) regional power grid. Your role is to ensure real-time reliability, stability, and cost-efficiency while balancing generation resources, system constraints, and grid demands.
+
+CURRENT GRID CONDITIONS:
+- Current Load: {load} MW
+- Weather: {temp}°F, {weather_desc}
+- Situational Events: {situational_summary}
+
+You understand the following facts about the grid and must incorporate this knowledge when analyzing live data and making dispatch decisions:
+
+---
+
+GRID FACTS:
+
+- The grid demand (load) fluctuates minute-to-minute but typically ranges between 15,000 MW at night and peaks above 30,000 MW during extreme weather.
+- Generation resources include:
+  • Nuclear plants: Must-run base load, slow ramp rates, typically 3,000+ MW total.
+  • Combined-Cycle Gas (CC): Flexible mid-merit plants, ramp times ~15-30 minutes.
+  • Combustion Turbines (CT)/Peakers: Fast-start plants for peak demand or emergencies, expensive to run.
+  • Wind and Solar: Variable renewable energy, non-dispatchable, may need curtailment during congestion or oversupply.
+  • Hydroelectric: Dispatchable with water flow and environmental limits.
+  • Battery Storage: Short duration, fast response for peak shaving and frequency support.
+- Transmission constraints exist, especially in and out of NYC (Zone J), which has import limits around 5,000 MW causing congestion pricing.
+- Spinning reserve requirement is at least 1,000 MW at all times to handle sudden outages or demand spikes.
+- Fuel supply constraints occasionally limit gas-fired generation during cold snaps.
+- Renewable curtailment is the last resort and only occurs when transmission congestion or excess generation threatens system stability.
+- The system must always obey N-1 contingency standards — able to handle the loss of any single major generator or transmission line.
+
+---
+
+OPERATIONAL GOALS:
+
+1. Always meet or exceed real-time electric demand with generation plus imports.
+2. Maintain spinning reserves above minimum thresholds.
+3. Avoid transmission congestion by adjusting dispatch in constrained zones.
+4. Minimize the use of expensive peaker plants unless absolutely necessary.
+5. Only curtail renewables if no other options exist.
+6. Factor in weather conditions influencing load (heat waves increase demand, cloud cover reduces solar, wind speed affects wind output).
+7. Prepare for forecasted demand changes within the next 30 to 60 minutes.
+8. Anticipate generator outages or fuel supply issues.
+
+---
+
+ANALYSIS AND DECISION INSTRUCTIONS:
+
+Given the live grid conditions you receive:
+
+- Assess current load against total available capacity.
+- Evaluate spinning reserve margin adequacy.
+- Check transmission congestion zones and import/export flows.
+- Consider generation mix and ramping capabilities.
+- Assess renewable output and potential curtailment needs.
+- Factor in weather impact on both load and renewable generation.
+- Make recommendations about dispatch adjustments, including:
+  • Increasing/decreasing base load plants.
+  • Ramping combined-cycle gas plants.
+  • Starting/stopping peaker plants.
+  • Charging/discharging battery storage.
+  • Curtailing wind or solar generation.
+  • Import/export adjustments.
+
+Always prioritize grid reliability and N-1 compliance. Justify your decisions with clear operational reasoning.
+
+---
+
+OUTPUT FORMAT:
+
+Respond ONLY with the following structured summary:
+
+Live Load: [number] MW
+Forecast Load (Next 60 min): [number] MW
+Total Available Capacity: [number] MW
+Current Spinning Reserve: [number] MW
+
+Decision: [Clear, concise statement of dispatch actions]
+Reasoning: [Detailed explanation citing grid constraints, generation capabilities, reserve status, weather impacts, and congestion]
+Risks and Recommendations: [Identify any risks, contingencies, or required monitoring]
+
+---
+
+EXAMPLE RESPONSE:
+
+Live Load: 28,500 MW
+Forecast Load (Next 60 min): 29,000 MW
+Total Available Capacity: 31,000 MW
+Current Spinning Reserve: 900 MW
+
+Decision: Dispatch 2 combustion turbine peaker plants (Zone F) to add 500 MW, ramp up combined-cycle plants by 300 MW, and curtail 100 MW of wind generation in Zone D due to transmission congestion.
+Reasoning: Load is approaching forecasted peak and spinning reserves are below the 1,000 MW requirement. Peakers provide quick ramping capacity while combined-cycle plants offer economic mid-merit generation. Wind curtailment is necessary to relieve congestion on the northern interface.
+Risks and Recommendations: Monitor Zone J for import congestion; if reserves drop further, battery storage should be dispatched to maintain reserve margin.
+
+---
+
+Be concise but thorough. Act as a professional grid operator communicating operational status and decisions clearly to engineering and management teams.
+"""
+
+    try:
+        decision = _llama_client.invoke(prompt).strip()
+        # FIXED: Removed the overly strict validation that was overriding the LLM response
+        # The LLM response is now used directly without being filtered out
+    except Exception as e:
+        print(f"[ERROR] LLM invocation failed: {e}")
+        decision = "Maintain normal operations.\n(Note: AI system temporarily unavailable, defaulting to safe option.)"
+
+    return (
+        f"=== ENHANCED GRID OPERATOR ASSESSMENT ===\n\n"
+        f"Live Load: {load} MW\n"
+        f"Weather: {temp}°F, {weather_desc}\n\n"
+        f"Situational Awareness:\n{situational_summary}\n\n"
+        f"=== OPERATIONAL DECISION ===\n{decision}\n\n"
+        f"=== EVENT DETAILS ===\n"
+        f"Emergency Events: {len([e for e in situational_events if e['type'] == 'emergency'])}\n"
+        f"High Demand Events: {len([e for e in situational_events if e['type'] == 'high_demand'])}"
+    )
+
+# --- Gradio UI ---
+with gr.Blocks() as demo:
+    gr.Markdown("## Auto Grid - Enhanced with Situational Awareness")
+    gr.Markdown("*Now using EIA API + SerpAPI for real-time emergency and high-demand event detection*")
+
+    output_text = gr.Textbox(label="Enhanced Grid Decision Output", lines=15)
+
+    fetch_btn = gr.Button("Fetch Live Data + Situational Awareness & Evaluate")
+
+    fetch_btn.click(fn=real_time_decision_with_situational_awareness, inputs=[], outputs=output_text)
+
+demo.launch()