Spaces:
Sleeping
Sleeping
| import os | |
| import argparse | |
| import logging | |
| import pickle | |
| import threading | |
| import time | |
| from datetime import datetime, timedelta | |
| from collections import defaultdict | |
| import csv | |
| import gradio as gr | |
| import pandas as pd | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| import matplotlib.animation as animation | |
| from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas | |
| import cartopy.crs as ccrs | |
| import cartopy.feature as cfeature | |
| import plotly.graph_objects as go | |
| import plotly.express as px | |
| from plotly.subplots import make_subplots | |
| from sklearn.manifold import TSNE | |
| from sklearn.cluster import DBSCAN | |
| from sklearn.preprocessing import StandardScaler | |
| from scipy.interpolate import interp1d | |
| import statsmodels.api as sm | |
| import requests | |
| import tempfile | |
| import shutil | |
| import xarray as xr | |
| try: | |
| import cdsapi | |
| CDSAPI_AVAILABLE = True | |
| except ImportError: | |
| CDSAPI_AVAILABLE = False | |
| import tropycal.tracks as tracks | |
| # ----------------------------- | |
| # Configuration and Setup | |
| # ----------------------------- | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - %(levelname)s - %(message)s' | |
| ) | |
| parser = argparse.ArgumentParser(description='Typhoon Analysis Dashboard') | |
| parser.add_argument('--data_path', type=str, default=os.getcwd(), help='Path to the data directory') | |
| args = parser.parse_args() | |
| # Enhanced data path handling for HuggingFace Spaces | |
| if 'SPACE_ID' in os.environ: | |
| # Running on HuggingFace Spaces | |
| DATA_PATH = '/tmp/typhoon_data' | |
| os.makedirs(DATA_PATH, exist_ok=True) | |
| logging.info(f"Running on HuggingFace Spaces, using data path: {DATA_PATH}") | |
| else: | |
| # Local development | |
| DATA_PATH = os.environ.get('DATA_PATH', tempfile.gettempdir()) | |
| # Ensure directory exists and is writable | |
| try: | |
| os.makedirs(DATA_PATH, exist_ok=True) | |
| # Test write permissions | |
| test_file = os.path.join(DATA_PATH, 'test_write.txt') | |
| with open(test_file, 'w') as f: | |
| f.write('test') | |
| os.remove(test_file) | |
| logging.info(f"Data directory is writable: {DATA_PATH}") | |
| except Exception as e: | |
| logging.warning(f"Data directory not writable, using temp dir: {e}") | |
| DATA_PATH = tempfile.mkdtemp() | |
| logging.info(f"Using temporary directory: {DATA_PATH}") | |
| # Update file paths | |
| ONI_DATA_PATH = os.path.join(DATA_PATH, 'oni_data.csv') | |
| TYPHOON_DATA_PATH = os.path.join(DATA_PATH, 'processed_typhoon_data.csv') | |
| MERGED_DATA_CSV = os.path.join(DATA_PATH, 'merged_typhoon_era5_data.csv') | |
| # IBTrACS settings | |
| BASIN_FILES = { | |
| 'EP': 'ibtracs.EP.list.v04r01.csv', | |
| 'NA': 'ibtracs.NA.list.v04r01.csv', | |
| 'WP': 'ibtracs.WP.list.v04r01.csv' | |
| } | |
| IBTRACS_BASE_URL = 'https://www.ncei.noaa.gov/data/international-best-track-archive-for-climate-stewardship-ibtracs/v04r01/access/csv/' | |
| LOCAL_IBTRACS_PATH = os.path.join(DATA_PATH, 'ibtracs.WP.list.v04r01.csv') | |
| CACHE_FILE = os.path.join(DATA_PATH, 'ibtracs_cache.pkl') | |
| CACHE_EXPIRY_DAYS = 1 | |
| # ----------------------------- | |
| # Color Maps and Standards | |
| # ----------------------------- | |
| color_map = { | |
| 'C5 Super Typhoon': 'rgb(255, 0, 0)', | |
| 'C4 Very Strong Typhoon': 'rgb(255, 165, 0)', | |
| 'C3 Strong Typhoon': 'rgb(255, 255, 0)', | |
| 'C2 Typhoon': 'rgb(0, 255, 0)', | |
| 'C1 Typhoon': 'rgb(0, 255, 255)', | |
| 'Tropical Storm': 'rgb(0, 0, 255)', | |
| 'Tropical Depression': 'rgb(128, 128, 128)' | |
| } | |
| atlantic_standard = { | |
| 'C5 Super Typhoon': {'wind_speed': 137, 'color': 'Red', 'hex': '#FF0000'}, | |
| 'C4 Very Strong Typhoon': {'wind_speed': 113, 'color': 'Orange', 'hex': '#FFA500'}, | |
| 'C3 Strong Typhoon': {'wind_speed': 96, 'color': 'Yellow', 'hex': '#FFFF00'}, | |
| 'C2 Typhoon': {'wind_speed': 83, 'color': 'Green', 'hex': '#00FF00'}, | |
| 'C1 Typhoon': {'wind_speed': 64, 'color': 'Cyan', 'hex': '#00FFFF'}, | |
| 'Tropical Storm': {'wind_speed': 34, 'color': 'Blue', 'hex': '#0000FF'}, | |
| 'Tropical Depression': {'wind_speed': 0, 'color': 'Gray', 'hex': '#808080'} | |
| } | |
| taiwan_standard = { | |
| 'Strong Typhoon': {'wind_speed': 51.0, 'color': 'Red', 'hex': '#FF0000'}, | |
| 'Medium Typhoon': {'wind_speed': 33.7, 'color': 'Orange', 'hex': '#FFA500'}, | |
| 'Mild Typhoon': {'wind_speed': 17.2, 'color': 'Yellow', 'hex': '#FFFF00'}, | |
| 'Tropical Depression': {'wind_speed': 0, 'color': 'Gray', 'hex': '#808080'} | |
| } | |
| # ----------------------------- | |
| # Season and Regions | |
| # ----------------------------- | |
| season_months = { | |
| 'all': list(range(1, 13)), | |
| 'summer': [6, 7, 8], | |
| 'winter': [12, 1, 2] | |
| } | |
| regions = { | |
| "Taiwan Land": {"lat_min": 21.8, "lat_max": 25.3, "lon_min": 119.5, "lon_max": 122.1}, | |
| "Taiwan Sea": {"lat_min": 19, "lat_max": 28, "lon_min": 117, "lon_max": 125}, | |
| "Japan": {"lat_min": 20, "lat_max": 45, "lon_min": 120, "lon_max": 150}, | |
| "China": {"lat_min": 18, "lat_max": 53, "lon_min": 73, "lon_max": 135}, | |
| "Hong Kong": {"lat_min": 21.5, "lat_max": 23, "lon_min": 113, "lon_max": 115}, | |
| "Philippines": {"lat_min": 5, "lat_max": 21, "lon_min": 115, "lon_max": 130} | |
| } | |
| # ----------------------------- | |
| # Utility Functions for HF Spaces | |
| # ----------------------------- | |
| def safe_file_write(file_path, data_frame, backup_dir=None): | |
| """Safely write DataFrame to CSV with backup and error handling""" | |
| try: | |
| # Create directory if it doesn't exist | |
| os.makedirs(os.path.dirname(file_path), exist_ok=True) | |
| # Try to write to a temporary file first | |
| temp_path = file_path + '.tmp' | |
| data_frame.to_csv(temp_path, index=False) | |
| # If successful, rename to final file | |
| os.rename(temp_path, file_path) | |
| logging.info(f"Successfully saved {len(data_frame)} records to {file_path}") | |
| return True | |
| except PermissionError as e: | |
| logging.warning(f"Permission denied writing to {file_path}: {e}") | |
| if backup_dir: | |
| try: | |
| backup_path = os.path.join(backup_dir, os.path.basename(file_path)) | |
| data_frame.to_csv(backup_path, index=False) | |
| logging.info(f"Saved to backup location: {backup_path}") | |
| return True | |
| except Exception as backup_e: | |
| logging.error(f"Failed to save to backup location: {backup_e}") | |
| return False | |
| except Exception as e: | |
| logging.error(f"Error saving file {file_path}: {e}") | |
| # Clean up temp file if it exists | |
| if os.path.exists(temp_path): | |
| try: | |
| os.remove(temp_path) | |
| except: | |
| pass | |
| return False | |
| def get_fallback_data_dir(): | |
| """Get a fallback data directory that's guaranteed to be writable""" | |
| fallback_dirs = [ | |
| tempfile.gettempdir(), | |
| '/tmp', | |
| os.path.expanduser('~'), | |
| os.getcwd() | |
| ] | |
| for directory in fallback_dirs: | |
| try: | |
| test_dir = os.path.join(directory, 'typhoon_fallback') | |
| os.makedirs(test_dir, exist_ok=True) | |
| test_file = os.path.join(test_dir, 'test.txt') | |
| with open(test_file, 'w') as f: | |
| f.write('test') | |
| os.remove(test_file) | |
| return test_dir | |
| except: | |
| continue | |
| # If all else fails, use current directory | |
| return os.getcwd() | |
| # ----------------------------- | |
| # ONI and Typhoon Data Functions | |
| # ----------------------------- | |
| def download_oni_file(url, filename): | |
| """Download ONI file with retry logic""" | |
| max_retries = 3 | |
| for attempt in range(max_retries): | |
| try: | |
| response = requests.get(url, timeout=30) | |
| response.raise_for_status() | |
| with open(filename, 'wb') as f: | |
| f.write(response.content) | |
| return True | |
| except Exception as e: | |
| logging.warning(f"Attempt {attempt + 1} failed to download ONI: {e}") | |
| if attempt < max_retries - 1: | |
| time.sleep(2 ** attempt) # Exponential backoff | |
| else: | |
| logging.error(f"Failed to download ONI after {max_retries} attempts") | |
| return False | |
| def convert_oni_ascii_to_csv(input_file, output_file): | |
| """Convert ONI ASCII format to CSV""" | |
| data = defaultdict(lambda: [''] * 12) | |
| season_to_month = {'DJF':12, 'JFM':1, 'FMA':2, 'MAM':3, 'AMJ':4, 'MJJ':5, | |
| 'JJA':6, 'JAS':7, 'ASO':8, 'SON':9, 'OND':10, 'NDJ':11} | |
| try: | |
| with open(input_file, 'r') as f: | |
| lines = f.readlines()[1:] # Skip header | |
| for line in lines: | |
| parts = line.split() | |
| if len(parts) >= 4: | |
| season, year, anom = parts[0], parts[1], parts[-1] | |
| if season in season_to_month: | |
| month = season_to_month[season] | |
| if season == 'DJF': | |
| year = str(int(year)-1) | |
| data[year][month-1] = anom | |
| # Write to CSV with safe write | |
| df = pd.DataFrame(data).T.reset_index() | |
| df.columns = ['Year','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] | |
| df = df.sort_values('Year').reset_index(drop=True) | |
| return safe_file_write(output_file, df, get_fallback_data_dir()) | |
| except Exception as e: | |
| logging.error(f"Error converting ONI file: {e}") | |
| return False | |
| def update_oni_data(): | |
| """Update ONI data with error handling""" | |
| url = "https://www.cpc.ncep.noaa.gov/data/indices/oni.ascii.txt" | |
| temp_file = os.path.join(DATA_PATH, "temp_oni.ascii.txt") | |
| input_file = os.path.join(DATA_PATH, "oni.ascii.txt") | |
| output_file = ONI_DATA_PATH | |
| try: | |
| if download_oni_file(url, temp_file): | |
| if not os.path.exists(input_file) or not os.path.exists(output_file): | |
| os.rename(temp_file, input_file) | |
| convert_oni_ascii_to_csv(input_file, output_file) | |
| else: | |
| os.remove(temp_file) | |
| else: | |
| # Create fallback ONI data if download fails | |
| logging.warning("Creating fallback ONI data") | |
| create_fallback_oni_data(output_file) | |
| except Exception as e: | |
| logging.error(f"Error updating ONI data: {e}") | |
| create_fallback_oni_data(output_file) | |
| def create_fallback_oni_data(output_file): | |
| """Create minimal ONI data for testing""" | |
| years = range(2000, 2025) | |
| months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] | |
| # Create synthetic ONI data | |
| data = [] | |
| for year in years: | |
| row = [year] | |
| for month in months: | |
| # Generate some realistic ONI values | |
| value = np.random.normal(0, 1) * 0.5 | |
| row.append(f"{value:.2f}") | |
| data.append(row) | |
| df = pd.DataFrame(data, columns=['Year'] + months) | |
| safe_file_write(output_file, df, get_fallback_data_dir()) | |
| # ----------------------------- | |
| # FIXED: IBTrACS Data Loading | |
| # ----------------------------- | |
| def download_ibtracs_file(basin, force_download=False): | |
| """Download specific basin file from IBTrACS""" | |
| filename = BASIN_FILES[basin] | |
| local_path = os.path.join(DATA_PATH, filename) | |
| url = IBTRACS_BASE_URL + filename | |
| # Check if file exists and is recent (less than 7 days old) | |
| if os.path.exists(local_path) and not force_download: | |
| file_age = time.time() - os.path.getmtime(local_path) | |
| if file_age < 7 * 24 * 3600: # 7 days | |
| logging.info(f"Using cached {basin} basin file") | |
| return local_path | |
| try: | |
| logging.info(f"Downloading {basin} basin file from {url}") | |
| response = requests.get(url, timeout=60) | |
| response.raise_for_status() | |
| # Ensure directory exists | |
| os.makedirs(os.path.dirname(local_path), exist_ok=True) | |
| with open(local_path, 'wb') as f: | |
| f.write(response.content) | |
| logging.info(f"Successfully downloaded {basin} basin file") | |
| return local_path | |
| except Exception as e: | |
| logging.error(f"Failed to download {basin} basin file: {e}") | |
| return None | |
| def examine_ibtracs_structure(file_path): | |
| """Examine the actual structure of an IBTrACS CSV file""" | |
| try: | |
| with open(file_path, 'r') as f: | |
| lines = f.readlines() | |
| # Show first 5 lines | |
| logging.info("First 5 lines of IBTrACS file:") | |
| for i, line in enumerate(lines[:5]): | |
| logging.info(f"Line {i}: {line.strip()}") | |
| # The first line contains the actual column headers | |
| # No need to skip rows for IBTrACS v04r01 | |
| df = pd.read_csv(file_path, nrows=5) | |
| logging.info(f"Columns from first row: {list(df.columns)}") | |
| return list(df.columns) | |
| except Exception as e: | |
| logging.error(f"Error examining IBTrACS structure: {e}") | |
| return None | |
| def load_ibtracs_csv_directly(basin='WP'): | |
| """Load IBTrACS data directly from CSV - FIXED VERSION""" | |
| filename = BASIN_FILES[basin] | |
| local_path = os.path.join(DATA_PATH, filename) | |
| # Download if not exists | |
| if not os.path.exists(local_path): | |
| downloaded_path = download_ibtracs_file(basin) | |
| if not downloaded_path: | |
| return None | |
| try: | |
| # First, examine the structure | |
| actual_columns = examine_ibtracs_structure(local_path) | |
| if not actual_columns: | |
| logging.error("Could not examine IBTrACS file structure") | |
| return None | |
| # Read IBTrACS CSV - DON'T skip any rows for v04r01 | |
| # The first row contains proper column headers | |
| logging.info(f"Reading IBTrACS CSV file: {local_path}") | |
| df = pd.read_csv(local_path, low_memory=False) # Don't skip any rows | |
| logging.info(f"Original columns: {list(df.columns)}") | |
| logging.info(f"Data shape before cleaning: {df.shape}") | |
| # Check which essential columns exist | |
| required_cols = ['SID', 'ISO_TIME', 'LAT', 'LON'] | |
| available_required = [col for col in required_cols if col in df.columns] | |
| if len(available_required) < 2: | |
| logging.error(f"Missing critical columns. Available: {list(df.columns)}") | |
| return None | |
| # Clean and standardize the data | |
| if 'ISO_TIME' in df.columns: | |
| df['ISO_TIME'] = pd.to_datetime(df['ISO_TIME'], errors='coerce') | |
| # Clean numeric columns | |
| numeric_columns = ['LAT', 'LON', 'WMO_WIND', 'WMO_PRES', 'USA_WIND', 'USA_PRES'] | |
| for col in numeric_columns: | |
| if col in df.columns: | |
| df[col] = pd.to_numeric(df[col], errors='coerce') | |
| # Filter out invalid/missing critical data | |
| valid_rows = df['LAT'].notna() & df['LON'].notna() | |
| df = df[valid_rows] | |
| # Ensure LAT/LON are in reasonable ranges | |
| df = df[(df['LAT'] >= -90) & (df['LAT'] <= 90)] | |
| df = df[(df['LON'] >= -180) & (df['LON'] <= 180)] | |
| # Add basin info if missing | |
| if 'BASIN' not in df.columns: | |
| df['BASIN'] = basin | |
| # Add default columns if missing | |
| if 'NAME' not in df.columns: | |
| df['NAME'] = 'UNNAMED' | |
| if 'SEASON' not in df.columns and 'ISO_TIME' in df.columns: | |
| df['SEASON'] = df['ISO_TIME'].dt.year | |
| logging.info(f"Successfully loaded {len(df)} records from {basin} basin") | |
| return df | |
| except Exception as e: | |
| logging.error(f"Error reading IBTrACS CSV file: {e}") | |
| return None | |
| def load_ibtracs_data_fixed(): | |
| """Fixed version of IBTrACS data loading""" | |
| ibtracs_data = {} | |
| # Try to load each basin, but prioritize WP for this application | |
| load_order = ['WP', 'EP', 'NA'] | |
| for basin in load_order: | |
| try: | |
| logging.info(f"Loading {basin} basin data...") | |
| df = load_ibtracs_csv_directly(basin) | |
| if df is not None and not df.empty: | |
| ibtracs_data[basin] = df | |
| logging.info(f"Successfully loaded {basin} basin with {len(df)} records") | |
| else: | |
| logging.warning(f"No data loaded for basin {basin}") | |
| ibtracs_data[basin] = None | |
| except Exception as e: | |
| logging.error(f"Failed to load basin {basin}: {e}") | |
| ibtracs_data[basin] = None | |
| return ibtracs_data | |
| def load_data_fixed(oni_path, typhoon_path): | |
| """Fixed version of load_data function""" | |
| # Load ONI data | |
| oni_data = pd.DataFrame({'Year': [], 'Jan': [], 'Feb': [], 'Mar': [], 'Apr': [], | |
| 'May': [], 'Jun': [], 'Jul': [], 'Aug': [], 'Sep': [], | |
| 'Oct': [], 'Nov': [], 'Dec': []}) | |
| if not os.path.exists(oni_path): | |
| logging.warning(f"ONI data file not found: {oni_path}") | |
| update_oni_data() | |
| try: | |
| oni_data = pd.read_csv(oni_path) | |
| logging.info(f"Successfully loaded ONI data with {len(oni_data)} years") | |
| except Exception as e: | |
| logging.error(f"Error loading ONI data: {e}") | |
| update_oni_data() | |
| try: | |
| oni_data = pd.read_csv(oni_path) | |
| except Exception as e: | |
| logging.error(f"Still can't load ONI data: {e}") | |
| # Load typhoon data - NEW APPROACH | |
| typhoon_data = None | |
| # First, try to load from existing processed file | |
| if os.path.exists(typhoon_path): | |
| try: | |
| typhoon_data = pd.read_csv(typhoon_path, low_memory=False) | |
| # Ensure basic columns exist and are valid | |
| required_cols = ['LAT', 'LON'] | |
| if all(col in typhoon_data.columns for col in required_cols): | |
| if 'ISO_TIME' in typhoon_data.columns: | |
| typhoon_data['ISO_TIME'] = pd.to_datetime(typhoon_data['ISO_TIME'], errors='coerce') | |
| logging.info(f"Loaded processed typhoon data with {len(typhoon_data)} records") | |
| else: | |
| logging.warning("Processed typhoon data missing required columns, will reload from IBTrACS") | |
| typhoon_data = None | |
| except Exception as e: | |
| logging.error(f"Error loading processed typhoon data: {e}") | |
| typhoon_data = None | |
| # If no valid processed data, load from IBTrACS | |
| if typhoon_data is None or typhoon_data.empty: | |
| logging.info("Loading typhoon data from IBTrACS...") | |
| ibtracs_data = load_ibtracs_data_fixed() | |
| # Combine all available basin data, prioritizing WP | |
| combined_dfs = [] | |
| for basin in ['WP', 'EP', 'NA']: | |
| if basin in ibtracs_data and ibtracs_data[basin] is not None: | |
| df = ibtracs_data[basin].copy() | |
| df['BASIN'] = basin | |
| combined_dfs.append(df) | |
| if combined_dfs: | |
| typhoon_data = pd.concat(combined_dfs, ignore_index=True) | |
| # Ensure SID has proper format | |
| if 'SID' not in typhoon_data.columns and 'BASIN' in typhoon_data.columns: | |
| # Create SID from basin and other identifiers if missing | |
| if 'SEASON' in typhoon_data.columns: | |
| typhoon_data['SID'] = (typhoon_data['BASIN'].astype(str) + | |
| typhoon_data.index.astype(str).str.zfill(2) + | |
| typhoon_data['SEASON'].astype(str)) | |
| else: | |
| typhoon_data['SID'] = (typhoon_data['BASIN'].astype(str) + | |
| typhoon_data.index.astype(str).str.zfill(2) + | |
| '2000') | |
| # Save the processed data for future use | |
| safe_file_write(typhoon_path, typhoon_data, get_fallback_data_dir()) | |
| logging.info(f"Combined IBTrACS data: {len(typhoon_data)} total records") | |
| else: | |
| logging.error("Failed to load any IBTrACS basin data") | |
| # Create minimal fallback data | |
| typhoon_data = create_fallback_typhoon_data() | |
| # Final validation of typhoon data | |
| if typhoon_data is not None: | |
| # Ensure required columns exist with fallback values | |
| required_columns = { | |
| 'SID': 'UNKNOWN', | |
| 'ISO_TIME': pd.Timestamp('2000-01-01'), | |
| 'LAT': 0.0, | |
| 'LON': 0.0, | |
| 'USA_WIND': np.nan, | |
| 'USA_PRES': np.nan, | |
| 'NAME': 'UNNAMED', | |
| 'SEASON': 2000 | |
| } | |
| for col, default_val in required_columns.items(): | |
| if col not in typhoon_data.columns: | |
| typhoon_data[col] = default_val | |
| logging.warning(f"Added missing column {col} with default value") | |
| # Ensure data types | |
| if 'ISO_TIME' in typhoon_data.columns: | |
| typhoon_data['ISO_TIME'] = pd.to_datetime(typhoon_data['ISO_TIME'], errors='coerce') | |
| typhoon_data['LAT'] = pd.to_numeric(typhoon_data['LAT'], errors='coerce') | |
| typhoon_data['LON'] = pd.to_numeric(typhoon_data['LON'], errors='coerce') | |
| typhoon_data['USA_WIND'] = pd.to_numeric(typhoon_data['USA_WIND'], errors='coerce') | |
| typhoon_data['USA_PRES'] = pd.to_numeric(typhoon_data['USA_PRES'], errors='coerce') | |
| # Remove rows with invalid coordinates | |
| typhoon_data = typhoon_data.dropna(subset=['LAT', 'LON']) | |
| logging.info(f"Final typhoon data: {len(typhoon_data)} records after validation") | |
| return oni_data, typhoon_data | |
| def create_fallback_typhoon_data(): | |
| """Create minimal fallback typhoon data - FIXED VERSION""" | |
| # Use proper pandas date_range instead of numpy | |
| dates = pd.date_range(start='2000-01-01', end='2023-12-31', freq='D') | |
| storm_dates = dates[np.random.choice(len(dates), size=100, replace=False)] | |
| data = [] | |
| for i, date in enumerate(storm_dates): | |
| # Create realistic WP storm tracks | |
| base_lat = np.random.uniform(10, 30) | |
| base_lon = np.random.uniform(130, 160) | |
| # Generate 20-50 data points per storm | |
| track_length = np.random.randint(20, 51) | |
| sid = f"WP{i+1:02d}{date.year}" | |
| for j in range(track_length): | |
| lat = base_lat + j * 0.2 + np.random.normal(0, 0.1) | |
| lon = base_lon + j * 0.3 + np.random.normal(0, 0.1) | |
| wind = max(25, 70 + np.random.normal(0, 20)) | |
| pres = max(950, 1000 - wind + np.random.normal(0, 5)) | |
| data.append({ | |
| 'SID': sid, | |
| 'ISO_TIME': date + pd.Timedelta(hours=j*6), # Use pd.Timedelta instead | |
| 'NAME': f'FALLBACK_{i+1}', | |
| 'SEASON': date.year, | |
| 'LAT': lat, | |
| 'LON': lon, | |
| 'USA_WIND': wind, | |
| 'USA_PRES': pres, | |
| 'BASIN': 'WP' | |
| }) | |
| df = pd.DataFrame(data) | |
| logging.info(f"Created fallback typhoon data with {len(df)} records") | |
| return df | |
| def process_oni_data(oni_data): | |
| """Process ONI data into long format""" | |
| oni_long = oni_data.melt(id_vars=['Year'], var_name='Month', value_name='ONI') | |
| month_map = {'Jan':'01','Feb':'02','Mar':'03','Apr':'04','May':'05','Jun':'06', | |
| 'Jul':'07','Aug':'08','Sep':'09','Oct':'10','Nov':'11','Dec':'12'} | |
| oni_long['Month'] = oni_long['Month'].map(month_map) | |
| oni_long['Date'] = pd.to_datetime(oni_long['Year'].astype(str)+'-'+oni_long['Month']+'-01') | |
| oni_long['ONI'] = pd.to_numeric(oni_long['ONI'], errors='coerce') | |
| return oni_long | |
| def process_typhoon_data(typhoon_data): | |
| """Process typhoon data""" | |
| if 'ISO_TIME' in typhoon_data.columns: | |
| typhoon_data['ISO_TIME'] = pd.to_datetime(typhoon_data['ISO_TIME'], errors='coerce') | |
| typhoon_data['USA_WIND'] = pd.to_numeric(typhoon_data['USA_WIND'], errors='coerce') | |
| typhoon_data['USA_PRES'] = pd.to_numeric(typhoon_data['USA_PRES'], errors='coerce') | |
| typhoon_data['LON'] = pd.to_numeric(typhoon_data['LON'], errors='coerce') | |
| logging.info(f"Unique basins in typhoon_data: {typhoon_data['SID'].str[:2].unique()}") | |
| typhoon_max = typhoon_data.groupby('SID').agg({ | |
| 'USA_WIND':'max','USA_PRES':'min','ISO_TIME':'first','SEASON':'first','NAME':'first', | |
| 'LAT':'first','LON':'first' | |
| }).reset_index() | |
| if 'ISO_TIME' in typhoon_max.columns: | |
| typhoon_max['Month'] = typhoon_max['ISO_TIME'].dt.strftime('%m') | |
| typhoon_max['Year'] = typhoon_max['ISO_TIME'].dt.year | |
| else: | |
| # Fallback if no ISO_TIME | |
| typhoon_max['Month'] = '01' | |
| typhoon_max['Year'] = typhoon_max['SEASON'] | |
| typhoon_max['Category'] = typhoon_max['USA_WIND'].apply(categorize_typhoon) | |
| return typhoon_max | |
| def merge_data(oni_long, typhoon_max): | |
| """Merge ONI and typhoon data""" | |
| return pd.merge(typhoon_max, oni_long, on=['Year','Month']) | |
| def categorize_typhoon(wind_speed): | |
| """Categorize typhoon based on wind speed""" | |
| if pd.isna(wind_speed): | |
| return 'Tropical Depression' | |
| if wind_speed >= 137: | |
| return 'C5 Super Typhoon' | |
| elif wind_speed >= 113: | |
| return 'C4 Very Strong Typhoon' | |
| elif wind_speed >= 96: | |
| return 'C3 Strong Typhoon' | |
| elif wind_speed >= 83: | |
| return 'C2 Typhoon' | |
| elif wind_speed >= 64: | |
| return 'C1 Typhoon' | |
| elif wind_speed >= 34: | |
| return 'Tropical Storm' | |
| else: | |
| return 'Tropical Depression' | |
| def classify_enso_phases(oni_value): | |
| """Classify ENSO phases based on ONI value""" | |
| if isinstance(oni_value, pd.Series): | |
| oni_value = oni_value.iloc[0] | |
| if pd.isna(oni_value): | |
| return 'Neutral' | |
| if oni_value >= 0.5: | |
| return 'El Nino' | |
| elif oni_value <= -0.5: | |
| return 'La Nina' | |
| else: | |
| return 'Neutral' | |
| # ----------------------------- | |
| # Regression Functions | |
| # ----------------------------- | |
| def perform_wind_regression(start_year, start_month, end_year, end_month): | |
| """Perform wind regression analysis""" | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].dropna(subset=['USA_WIND','ONI']) | |
| data['severe_typhoon'] = (data['USA_WIND']>=64).astype(int) | |
| X = sm.add_constant(data['ONI']) | |
| y = data['severe_typhoon'] | |
| try: | |
| model = sm.Logit(y, X).fit(disp=0) | |
| beta_1 = model.params['ONI'] | |
| exp_beta_1 = np.exp(beta_1) | |
| p_value = model.pvalues['ONI'] | |
| return f"Wind Regression: β1={beta_1:.4f}, Odds Ratio={exp_beta_1:.4f}, P-value={p_value:.4f}" | |
| except Exception as e: | |
| return f"Wind Regression Error: {e}" | |
| def perform_pressure_regression(start_year, start_month, end_year, end_month): | |
| """Perform pressure regression analysis""" | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].dropna(subset=['USA_PRES','ONI']) | |
| data['intense_typhoon'] = (data['USA_PRES']<=950).astype(int) | |
| X = sm.add_constant(data['ONI']) | |
| y = data['intense_typhoon'] | |
| try: | |
| model = sm.Logit(y, X).fit(disp=0) | |
| beta_1 = model.params['ONI'] | |
| exp_beta_1 = np.exp(beta_1) | |
| p_value = model.pvalues['ONI'] | |
| return f"Pressure Regression: β1={beta_1:.4f}, Odds Ratio={exp_beta_1:.4f}, P-value={p_value:.4f}" | |
| except Exception as e: | |
| return f"Pressure Regression Error: {e}" | |
| def perform_longitude_regression(start_year, start_month, end_year, end_month): | |
| """Perform longitude regression analysis""" | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].dropna(subset=['LON','ONI']) | |
| data['western_typhoon'] = (data['LON']<=140).astype(int) | |
| X = sm.add_constant(data['ONI']) | |
| y = data['western_typhoon'] | |
| try: | |
| model = sm.OLS(y, sm.add_constant(X)).fit() | |
| beta_1 = model.params['ONI'] | |
| exp_beta_1 = np.exp(beta_1) | |
| p_value = model.pvalues['ONI'] | |
| return f"Longitude Regression: β1={beta_1:.4f}, Odds Ratio={exp_beta_1:.4f}, P-value={p_value:.4f}" | |
| except Exception as e: | |
| return f"Longitude Regression Error: {e}" | |
| # ----------------------------- | |
| # Visualization Functions | |
| # ----------------------------- | |
| def generate_typhoon_tracks(filtered_data, typhoon_search): | |
| """Generate typhoon tracks visualization""" | |
| fig = go.Figure() | |
| for sid in filtered_data['SID'].unique(): | |
| storm_data = filtered_data[filtered_data['SID'] == sid] | |
| phase = storm_data['ENSO_Phase'].iloc[0] | |
| color = {'El Nino':'red','La Nina':'blue','Neutral':'green'}.get(phase, 'black') | |
| fig.add_trace(go.Scattergeo( | |
| lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines', | |
| name=storm_data['NAME'].iloc[0], line=dict(width=2, color=color) | |
| )) | |
| if typhoon_search: | |
| mask = filtered_data['NAME'].str.contains(typhoon_search, case=False, na=False) | |
| if mask.any(): | |
| storm_data = filtered_data[mask] | |
| fig.add_trace(go.Scattergeo( | |
| lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines', | |
| name=f'Matched: {typhoon_search}', line=dict(width=5, color='yellow') | |
| )) | |
| fig.update_layout( | |
| title='Typhoon Tracks', | |
| geo=dict(projection_type='natural earth', showland=True), | |
| height=700 | |
| ) | |
| return fig | |
| def generate_wind_oni_scatter(filtered_data, typhoon_search): | |
| """Generate wind vs ONI scatter plot""" | |
| fig = px.scatter(filtered_data, x='ONI', y='USA_WIND', color='Category', | |
| hover_data=['NAME','Year','Category'], | |
| title='Wind Speed vs ONI', | |
| labels={'ONI':'ONI Value','USA_WIND':'Max Wind Speed (knots)'}, | |
| color_discrete_map=color_map) | |
| if typhoon_search: | |
| mask = filtered_data['NAME'].str.contains(typhoon_search, case=False, na=False) | |
| if mask.any(): | |
| fig.add_trace(go.Scatter( | |
| x=filtered_data.loc[mask,'ONI'], y=filtered_data.loc[mask,'USA_WIND'], | |
| mode='markers', marker=dict(size=10, color='red', symbol='star'), | |
| name=f'Matched: {typhoon_search}', | |
| text=filtered_data.loc[mask,'NAME']+' ('+filtered_data.loc[mask,'Year'].astype(str)+')' | |
| )) | |
| return fig | |
| def generate_pressure_oni_scatter(filtered_data, typhoon_search): | |
| """Generate pressure vs ONI scatter plot""" | |
| fig = px.scatter(filtered_data, x='ONI', y='USA_PRES', color='Category', | |
| hover_data=['NAME','Year','Category'], | |
| title='Pressure vs ONI', | |
| labels={'ONI':'ONI Value','USA_PRES':'Min Pressure (hPa)'}, | |
| color_discrete_map=color_map) | |
| if typhoon_search: | |
| mask = filtered_data['NAME'].str.contains(typhoon_search, case=False, na=False) | |
| if mask.any(): | |
| fig.add_trace(go.Scatter( | |
| x=filtered_data.loc[mask,'ONI'], y=filtered_data.loc[mask,'USA_PRES'], | |
| mode='markers', marker=dict(size=10, color='red', symbol='star'), | |
| name=f'Matched: {typhoon_search}', | |
| text=filtered_data.loc[mask,'NAME']+' ('+filtered_data.loc[mask,'Year'].astype(str)+')' | |
| )) | |
| return fig | |
| def generate_regression_analysis(filtered_data): | |
| """Generate regression analysis plot""" | |
| fig = px.scatter(filtered_data, x='LON', y='ONI', hover_data=['NAME'], | |
| title='Typhoon Generation Longitude vs ONI (All Years)') | |
| if len(filtered_data) > 1: | |
| X = np.array(filtered_data['LON']).reshape(-1,1) | |
| y = filtered_data['ONI'] | |
| try: | |
| model = sm.OLS(y, sm.add_constant(X)).fit() | |
| y_pred = model.predict(sm.add_constant(X)) | |
| fig.add_trace(go.Scatter(x=filtered_data['LON'], y=y_pred, mode='lines', name='Regression Line')) | |
| slope = model.params[1] | |
| slopes_text = f"All Years Slope: {slope:.4f}" | |
| except Exception as e: | |
| slopes_text = f"Regression Error: {e}" | |
| else: | |
| slopes_text = "Insufficient data for regression" | |
| return fig, slopes_text | |
| def generate_main_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search): | |
| """Generate main analysis plots""" | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| filtered_data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].copy() | |
| filtered_data['ENSO_Phase'] = filtered_data['ONI'].apply(classify_enso_phases) | |
| if enso_phase != 'all': | |
| filtered_data = filtered_data[filtered_data['ENSO_Phase'] == enso_phase.capitalize()] | |
| tracks_fig = generate_typhoon_tracks(filtered_data, typhoon_search) | |
| wind_scatter = generate_wind_oni_scatter(filtered_data, typhoon_search) | |
| pressure_scatter = generate_pressure_oni_scatter(filtered_data, typhoon_search) | |
| regression_fig, slopes_text = generate_regression_analysis(filtered_data) | |
| return tracks_fig, wind_scatter, pressure_scatter, regression_fig, slopes_text | |
| def get_full_tracks(start_year, start_month, end_year, end_month, enso_phase, typhoon_search): | |
| """Get full typhoon tracks""" | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| filtered_data = merged_data[(merged_data['ISO_TIME']>=start_date) & (merged_data['ISO_TIME']<=end_date)].copy() | |
| filtered_data['ENSO_Phase'] = filtered_data['ONI'].apply(classify_enso_phases) | |
| if enso_phase != 'all': | |
| filtered_data = filtered_data[filtered_data['ENSO_Phase'] == enso_phase.capitalize()] | |
| unique_storms = filtered_data['SID'].unique() | |
| count = len(unique_storms) | |
| fig = go.Figure() | |
| for sid in unique_storms: | |
| storm_data = typhoon_data[typhoon_data['SID']==sid] | |
| if storm_data.empty: | |
| continue | |
| name = storm_data['NAME'].iloc[0] if pd.notnull(storm_data['NAME'].iloc[0]) else "Unnamed" | |
| basin = storm_data['SID'].iloc[0][:2] | |
| storm_oni = filtered_data[filtered_data['SID']==sid]['ONI'].iloc[0] | |
| color = 'red' if storm_oni>=0.5 else ('blue' if storm_oni<=-0.5 else 'green') | |
| fig.add_trace(go.Scattergeo( | |
| lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines', | |
| name=f"{name} ({basin})", | |
| line=dict(width=1.5, color=color), hoverinfo="name" | |
| )) | |
| if typhoon_search: | |
| search_mask = typhoon_data['NAME'].str.contains(typhoon_search, case=False, na=False) | |
| if search_mask.any(): | |
| for sid in typhoon_data[search_mask]['SID'].unique(): | |
| storm_data = typhoon_data[typhoon_data['SID']==sid] | |
| fig.add_trace(go.Scattergeo( | |
| lon=storm_data['LON'], lat=storm_data['LAT'], mode='lines+markers', | |
| name=f"MATCHED: {storm_data['NAME'].iloc[0]}", | |
| line=dict(width=3, color='yellow'), | |
| marker=dict(size=5), hoverinfo="name" | |
| )) | |
| fig.update_layout( | |
| title=f"Typhoon Tracks ({start_year}-{start_month} to {end_year}-{end_month})", | |
| geo=dict( | |
| projection_type='natural earth', | |
| showland=True, | |
| showcoastlines=True, | |
| landcolor='rgb(243,243,243)', | |
| countrycolor='rgb(204,204,204)', | |
| coastlinecolor='rgb(204,204,204)', | |
| center=dict(lon=140, lat=20), | |
| projection_scale=3 | |
| ), | |
| legend_title="Typhoons by ENSO Phase", | |
| showlegend=True, | |
| height=700 | |
| ) | |
| fig.add_annotation( | |
| x=0.02, y=0.98, xref="paper", yref="paper", | |
| text="Red: El Niño, Blue: La Nina, Green: Neutral", | |
| showarrow=False, align="left", | |
| bgcolor="rgba(255,255,255,0.8)" | |
| ) | |
| return fig, f"Total typhoons displayed: {count}" | |
| def get_wind_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search): | |
| """Get wind analysis""" | |
| results = generate_main_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search) | |
| regression = perform_wind_regression(start_year, start_month, end_year, end_month) | |
| return results[1], regression | |
| def get_pressure_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search): | |
| """Get pressure analysis""" | |
| results = generate_main_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search) | |
| regression = perform_pressure_regression(start_year, start_month, end_year, end_month) | |
| return results[2], regression | |
| def get_longitude_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search): | |
| """Get longitude analysis""" | |
| results = generate_main_analysis(start_year, start_month, end_year, end_month, enso_phase, typhoon_search) | |
| regression = perform_longitude_regression(start_year, start_month, end_year, end_month) | |
| return results[3], results[4], regression | |
| def categorize_typhoon_by_standard(wind_speed, standard='atlantic'): | |
| """Categorize typhoon by standard""" | |
| if pd.isna(wind_speed): | |
| return 'Tropical Depression', '#808080' | |
| if standard=='taiwan': | |
| wind_speed_ms = wind_speed * 0.514444 | |
| if wind_speed_ms >= 51.0: | |
| return 'Strong Typhoon', taiwan_standard['Strong Typhoon']['hex'] | |
| elif wind_speed_ms >= 33.7: | |
| return 'Medium Typhoon', taiwan_standard['Medium Typhoon']['hex'] | |
| elif wind_speed_ms >= 17.2: | |
| return 'Mild Typhoon', taiwan_standard['Mild Typhoon']['hex'] | |
| return 'Tropical Depression', taiwan_standard['Tropical Depression']['hex'] | |
| else: | |
| if wind_speed >= 137: | |
| return 'C5 Super Typhoon', atlantic_standard['C5 Super Typhoon']['hex'] | |
| elif wind_speed >= 113: | |
| return 'C4 Very Strong Typhoon', atlantic_standard['C4 Very Strong Typhoon']['hex'] | |
| elif wind_speed >= 96: | |
| return 'C3 Strong Typhoon', atlantic_standard['C3 Strong Typhoon']['hex'] | |
| elif wind_speed >= 83: | |
| return 'C2 Typhoon', atlantic_standard['C2 Typhoon']['hex'] | |
| elif wind_speed >= 64: | |
| return 'C1 Typhoon', atlantic_standard['C1 Typhoon']['hex'] | |
| elif wind_speed >= 34: | |
| return 'Tropical Storm', atlantic_standard['Tropical Storm']['hex'] | |
| return 'Tropical Depression', atlantic_standard['Tropical Depression']['hex'] | |
| # ----------------------------- | |
| # TSNE Cluster Function | |
| # ----------------------------- | |
| def update_route_clusters(start_year, start_month, end_year, end_month, enso_value, season): | |
| """Updated TSNE cluster function with mean curves""" | |
| try: | |
| # Merge raw typhoon data with ONI | |
| raw_data = typhoon_data.copy() | |
| if 'ISO_TIME' not in raw_data.columns: | |
| logging.error("ISO_TIME column not found in typhoon data") | |
| return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "Error: ISO_TIME column missing" | |
| raw_data['Year'] = raw_data['ISO_TIME'].dt.year | |
| raw_data['Month'] = raw_data['ISO_TIME'].dt.strftime('%m') | |
| merged_raw = pd.merge(raw_data, process_oni_data(oni_data), on=['Year','Month'], how='left') | |
| # Filter by date | |
| start_date = datetime(start_year, start_month, 1) | |
| end_date = datetime(end_year, end_month, 28) | |
| merged_raw = merged_raw[(merged_raw['ISO_TIME'] >= start_date) & (merged_raw['ISO_TIME'] <= end_date)] | |
| logging.info(f"Total points after date filtering: {merged_raw.shape[0]}") | |
| # Filter by ENSO phase if specified | |
| merged_raw['ENSO_Phase'] = merged_raw['ONI'].apply(classify_enso_phases) | |
| if enso_value != 'all': | |
| merged_raw = merged_raw[merged_raw['ENSO_Phase'] == enso_value.capitalize()] | |
| logging.info(f"Total points after ENSO filtering: {merged_raw.shape[0]}") | |
| # Regional filtering for Western Pacific | |
| wp_data = merged_raw[(merged_raw['LON'] >= 100) & (merged_raw['LON'] <= 180) & | |
| (merged_raw['LAT'] >= 0) & (merged_raw['LAT'] <= 40)] | |
| logging.info(f"Total points after WP regional filtering: {wp_data.shape[0]}") | |
| if wp_data.empty: | |
| logging.info("WP regional filter returned no data; using all filtered data.") | |
| wp_data = merged_raw | |
| # Group by storm ID | |
| all_storms_data = [] | |
| for sid, group in wp_data.groupby('SID'): | |
| group = group.sort_values('ISO_TIME') | |
| times = pd.to_datetime(group['ISO_TIME']).values | |
| lats = group['LAT'].astype(float).values | |
| lons = group['LON'].astype(float).values | |
| if len(lons) < 2: | |
| continue | |
| # Extract wind and pressure curves | |
| wind = group['USA_WIND'].astype(float).values if 'USA_WIND' in group.columns else None | |
| pres = group['USA_PRES'].astype(float).values if 'USA_PRES' in group.columns else None | |
| all_storms_data.append((sid, lons, lats, times, wind, pres)) | |
| logging.info(f"Storms available for TSNE after grouping: {len(all_storms_data)}") | |
| if not all_storms_data: | |
| return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms for clustering." | |
| # Interpolate each storm's route to a common length | |
| max_length = min(50, max(len(item[1]) for item in all_storms_data)) # Cap at 50 points | |
| route_vectors = [] | |
| wind_curves = [] | |
| pres_curves = [] | |
| storm_ids = [] | |
| for sid, lons, lats, times, wind, pres in all_storms_data: | |
| t = np.linspace(0, 1, len(lons)) | |
| t_new = np.linspace(0, 1, max_length) | |
| try: | |
| lon_interp = interp1d(t, lons, kind='linear', fill_value='extrapolate')(t_new) | |
| lat_interp = interp1d(t, lats, kind='linear', fill_value='extrapolate')(t_new) | |
| except Exception as ex: | |
| logging.error(f"Interpolation error for storm {sid}: {ex}") | |
| continue | |
| route_vector = np.column_stack((lon_interp, lat_interp)).flatten() | |
| if np.isnan(route_vector).any(): | |
| continue | |
| route_vectors.append(route_vector) | |
| storm_ids.append(sid) | |
| # Interpolate wind and pressure | |
| if wind is not None and len(wind) >= 2: | |
| try: | |
| wind_interp = interp1d(t, wind, kind='linear', fill_value='extrapolate')(t_new) | |
| except Exception as ex: | |
| logging.error(f"Wind interpolation error for storm {sid}: {ex}") | |
| wind_interp = np.full(max_length, np.nan) | |
| else: | |
| wind_interp = np.full(max_length, np.nan) | |
| if pres is not None and len(pres) >= 2: | |
| try: | |
| pres_interp = interp1d(t, pres, kind='linear', fill_value='extrapolate')(t_new) | |
| except Exception as ex: | |
| logging.error(f"Pressure interpolation error for storm {sid}: {ex}") | |
| pres_interp = np.full(max_length, np.nan) | |
| else: | |
| pres_interp = np.full(max_length, np.nan) | |
| wind_curves.append(wind_interp) | |
| pres_curves.append(pres_interp) | |
| logging.info(f"Storms with valid route vectors: {len(route_vectors)}") | |
| if len(route_vectors) == 0: | |
| return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "No valid storms after interpolation." | |
| route_vectors = np.array(route_vectors) | |
| wind_curves = np.array(wind_curves) | |
| pres_curves = np.array(pres_curves) | |
| # Run TSNE on route vectors | |
| if len(route_vectors) < 5: | |
| return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), "Need at least 5 storms for clustering." | |
| tsne = TSNE(n_components=2, random_state=42, verbose=1, perplexity=min(30, len(route_vectors)-1)) | |
| tsne_results = tsne.fit_transform(route_vectors) | |
| # Dynamic DBSCAN | |
| selected_labels = None | |
| selected_eps = None | |
| for eps in np.linspace(1.0, 10.0, 91): | |
| dbscan = DBSCAN(eps=eps, min_samples=max(2, len(route_vectors)//10)) | |
| labels = dbscan.fit_predict(tsne_results) | |
| clusters = set(labels) - {-1} | |
| if 2 <= len(clusters) <= min(10, len(route_vectors)//2): | |
| selected_labels = labels | |
| selected_eps = eps | |
| break | |
| if selected_labels is None: | |
| selected_eps = 5.0 | |
| dbscan = DBSCAN(eps=selected_eps, min_samples=max(2, len(route_vectors)//10)) | |
| selected_labels = dbscan.fit_predict(tsne_results) | |
| logging.info(f"Selected DBSCAN eps: {selected_eps:.2f} yielding {len(set(selected_labels)-{-1})} clusters.") | |
| # TSNE scatter plot | |
| fig_tsne = go.Figure() | |
| colors = px.colors.qualitative.Set3 | |
| unique_labels = sorted(set(selected_labels) - {-1}) | |
| for i, label in enumerate(unique_labels): | |
| indices = np.where(selected_labels == label)[0] | |
| fig_tsne.add_trace(go.Scatter( | |
| x=tsne_results[indices, 0], | |
| y=tsne_results[indices, 1], | |
| mode='markers', | |
| marker=dict(color=colors[i % len(colors)]), | |
| name=f"Cluster {label}" | |
| )) | |
| noise_indices = np.where(selected_labels == -1)[0] | |
| if len(noise_indices) > 0: | |
| fig_tsne.add_trace(go.Scatter( | |
| x=tsne_results[noise_indices, 0], | |
| y=tsne_results[noise_indices, 1], | |
| mode='markers', | |
| marker=dict(color='grey'), | |
| name='Noise' | |
| )) | |
| fig_tsne.update_layout( | |
| title="t-SNE of Storm Routes", | |
| xaxis_title="t-SNE Dim 1", | |
| yaxis_title="t-SNE Dim 2" | |
| ) | |
| # Compute mean routes and curves for each cluster | |
| fig_routes = go.Figure() | |
| cluster_stats = [] | |
| for i, label in enumerate(unique_labels): | |
| indices = np.where(selected_labels == label)[0] | |
| cluster_ids = [storm_ids[j] for j in indices] | |
| cluster_vectors = route_vectors[indices, :] | |
| mean_vector = np.mean(cluster_vectors, axis=0) | |
| mean_route = mean_vector.reshape((max_length, 2)) | |
| mean_lon = mean_route[:, 0] | |
| mean_lat = mean_route[:, 1] | |
| fig_routes.add_trace(go.Scattergeo( | |
| lon=mean_lon, | |
| lat=mean_lat, | |
| mode='lines', | |
| line=dict(width=4, color=colors[i % len(colors)]), | |
| name=f"Cluster {label} Mean Route" | |
| )) | |
| # Compute mean curves | |
| cluster_winds = wind_curves[indices, :] | |
| cluster_pres = pres_curves[indices, :] | |
| mean_wind_curve = np.nanmean(cluster_winds, axis=0) | |
| mean_pres_curve = np.nanmean(cluster_pres, axis=0) | |
| cluster_stats.append((label, mean_wind_curve, mean_pres_curve)) | |
| fig_routes.update_layout( | |
| title="Cluster Mean Routes", | |
| geo=dict(projection_type='natural earth', showland=True), | |
| height=600 | |
| ) | |
| # Create cluster stats plot | |
| x_axis = np.linspace(0, 1, max_length) | |
| fig_stats = make_subplots(rows=2, cols=1, shared_xaxes=True, | |
| subplot_titles=("Mean Wind Speed (knots)", "Mean MSLP (hPa)")) | |
| for i, (label, wind_curve, pres_curve) in enumerate(cluster_stats): | |
| fig_stats.add_trace(go.Scatter( | |
| x=x_axis, | |
| y=wind_curve, | |
| mode='lines', | |
| line=dict(width=2, color=colors[i % len(colors)]), | |
| name=f"Cluster {label} Mean Wind", | |
| showlegend=True | |
| ), row=1, col=1) | |
| fig_stats.add_trace(go.Scatter( | |
| x=x_axis, | |
| y=pres_curve, | |
| mode='lines', | |
| line=dict(width=2, color=colors[i % len(colors)]), | |
| name=f"Cluster {label} Mean MSLP", | |
| showlegend=False | |
| ), row=2, col=1) | |
| fig_stats.update_layout( | |
| title="Cluster Mean Curves", | |
| xaxis_title="Normalized Route Index", | |
| yaxis_title="Mean Wind Speed (knots)", | |
| xaxis2_title="Normalized Route Index", | |
| yaxis2_title="Mean MSLP (hPa)", | |
| height=600 | |
| ) | |
| info = f"TSNE clustering complete. Selected eps: {selected_eps:.2f}. Clusters: {len(unique_labels)}. Total storms: {len(route_vectors)}." | |
| return fig_tsne, fig_routes, fig_stats, info | |
| except Exception as e: | |
| logging.error(f"Error in TSNE clustering: {e}") | |
| return go.Figure(), go.Figure(), make_subplots(rows=2, cols=1), f"Error in TSNE clustering: {e}" | |
| # ----------------------------- | |
| # Animation Functions | |
| # ----------------------------- | |
| def generate_track_video_from_csv(year, storm_id, standard): | |
| """Generate track video from CSV data""" | |
| storm_df = typhoon_data[typhoon_data['SID'] == storm_id].copy() | |
| if storm_df.empty: | |
| logging.error(f"No data found for storm: {storm_id}") | |
| return None | |
| storm_df = storm_df.sort_values('ISO_TIME') | |
| lats = storm_df['LAT'].astype(float).values | |
| lons = storm_df['LON'].astype(float).values | |
| times = pd.to_datetime(storm_df['ISO_TIME']).values | |
| if 'USA_WIND' in storm_df.columns: | |
| winds = pd.to_numeric(storm_df['USA_WIND'], errors='coerce').values | |
| else: | |
| winds = np.full(len(lats), np.nan) | |
| storm_name = storm_df['NAME'].iloc[0] if pd.notnull(storm_df['NAME'].iloc[0]) else "Unnamed" | |
| basin = storm_df['SID'].iloc[0][:2] | |
| season = storm_df['SEASON'].iloc[0] if 'SEASON' in storm_df.columns else year | |
| min_lat, max_lat = np.min(lats), np.max(lats) | |
| min_lon, max_lon = np.min(lons), np.max(lons) | |
| lat_padding = max((max_lat - min_lat)*0.3, 5) | |
| lon_padding = max((max_lon - min_lon)*0.3, 5) | |
| fig = plt.figure(figsize=(12,6), dpi=100) | |
| ax = plt.axes([0.05, 0.05, 0.60, 0.85], | |
| projection=ccrs.PlateCarree(central_longitude=180)) | |
| ax.stock_img() | |
| ax.set_extent([min_lon - lon_padding, max_lon + lon_padding, min_lat - lat_padding, max_lat + lat_padding], | |
| crs=ccrs.PlateCarree()) | |
| ax.coastlines(resolution='50m', color='black', linewidth=1) | |
| gl = ax.gridlines(draw_labels=True, color='gray', alpha=0.4, linestyle='--') | |
| gl.top_labels = gl.right_labels = False | |
| ax.set_title(f"{year} {storm_name} ({basin}) - {season}", fontsize=14) | |
| line, = ax.plot([], [], transform=ccrs.PlateCarree(), color='blue', linewidth=2) | |
| point, = ax.plot([], [], 'o', markersize=8, transform=ccrs.PlateCarree()) | |
| date_text = ax.text(0.02, 0.02, '', transform=ax.transAxes, fontsize=10, | |
| bbox=dict(facecolor='white', alpha=0.8)) | |
| storm_info_text = fig.text(0.70, 0.60, '', fontsize=10, | |
| bbox=dict(facecolor='white', alpha=0.8, boxstyle='round,pad=0.5')) | |
| from matplotlib.lines import Line2D | |
| standard_dict = atlantic_standard if standard=='atlantic' else taiwan_standard | |
| legend_elements = [Line2D([0],[0], marker='o', color='w', label=cat, | |
| markerfacecolor=details['hex'], markersize=8) | |
| for cat, details in standard_dict.items()] | |
| ax.legend(handles=legend_elements, title="Storm Categories", | |
| loc='upper right', fontsize=9) | |
| def init(): | |
| line.set_data([], []) | |
| point.set_data([], []) | |
| date_text.set_text('') | |
| storm_info_text.set_text('') | |
| return line, point, date_text, storm_info_text | |
| def update(frame): | |
| line.set_data(lons[:frame+1], lats[:frame+1]) | |
| point.set_data([lons[frame]], [lats[frame]]) | |
| wind_speed = winds[frame] if frame < len(winds) and not pd.isna(winds[frame]) else 0 | |
| category, color = categorize_typhoon_by_standard(wind_speed, standard) | |
| point.set_color(color) | |
| dt_str = pd.to_datetime(times[frame]).strftime('%Y-%m-%d %H:%M') | |
| date_text.set_text(dt_str) | |
| info_str = (f"Name: {storm_name}\nBasin: {basin}\nDate: {dt_str}\nWind: {wind_speed:.1f} kt\nCategory: {category}") | |
| storm_info_text.set_text(info_str) | |
| return line, point, date_text, storm_info_text | |
| ani = animation.FuncAnimation(fig, update, init_func=init, frames=len(times), | |
| interval=200, blit=True, repeat=True) | |
| # Create animation file | |
| temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4', dir=DATA_PATH) | |
| try: | |
| writer = animation.FFMpegWriter(fps=5, bitrate=1800) | |
| ani.save(temp_file.name, writer=writer) | |
| plt.close(fig) | |
| return temp_file.name | |
| except Exception as e: | |
| logging.error(f"Error creating animation: {e}") | |
| plt.close(fig) | |
| return None | |
| def simplified_track_video(year, basin, typhoon, standard): | |
| """Simplified track video function""" | |
| if not typhoon: | |
| return None | |
| storm_id = typhoon.split('(')[-1].strip(')') | |
| return generate_track_video_from_csv(year, storm_id, standard) | |
| # ----------------------------- | |
| # FIXED: Update Typhoon Options Function | |
| # ----------------------------- | |
| def update_typhoon_options_fixed(year, basin): | |
| """Fixed version of update_typhoon_options""" | |
| try: | |
| # Use the typhoon_data already loaded | |
| if typhoon_data is None or typhoon_data.empty: | |
| logging.error("No typhoon data available") | |
| return gr.update(choices=[], value=None) | |
| # Filter by year | |
| if 'ISO_TIME' in typhoon_data.columns: | |
| year_data = typhoon_data[typhoon_data['ISO_TIME'].dt.year == int(year)].copy() | |
| elif 'SEASON' in typhoon_data.columns: | |
| year_data = typhoon_data[typhoon_data['SEASON'] == int(year)].copy() | |
| else: | |
| # Fallback: use all data | |
| year_data = typhoon_data.copy() | |
| if basin != "All Basins": | |
| # Extract basin code | |
| basin_code = basin.split(' - ')[0] if ' - ' in basin else basin[:2] | |
| # Filter by basin | |
| if 'SID' in year_data.columns: | |
| year_data = year_data[year_data['SID'].str.startswith(basin_code, na=False)] | |
| elif 'BASIN' in year_data.columns: | |
| year_data = year_data[year_data['BASIN'] == basin_code] | |
| if year_data.empty: | |
| logging.warning(f"No storms found for year {year} and basin {basin}") | |
| return gr.update(choices=[], value=None) | |
| # Get unique storms and create options | |
| storms = year_data.groupby('SID').first().reset_index() | |
| options = [] | |
| for _, storm in storms.iterrows(): | |
| name = storm.get('NAME', 'UNNAMED') | |
| if pd.isna(name) or name == '' or name == 'UNNAMED': | |
| name = 'UNNAMED' | |
| sid = storm['SID'] | |
| options.append(f"{name} ({sid})") | |
| if not options: | |
| return gr.update(choices=[], value=None) | |
| return gr.update(choices=sorted(options), value=options[0]) | |
| except Exception as e: | |
| logging.error(f"Error in update_typhoon_options_fixed: {e}") | |
| return gr.update(choices=[], value=None) | |
| # ----------------------------- | |
| # Load & Process Data (using fixed functions) | |
| # ----------------------------- | |
| logging.info("Starting data loading process...") | |
| update_oni_data() | |
| oni_data, typhoon_data = load_data_fixed(ONI_DATA_PATH, TYPHOON_DATA_PATH) | |
| oni_long = process_oni_data(oni_data) | |
| typhoon_max = process_typhoon_data(typhoon_data) | |
| merged_data = merge_data(oni_long, typhoon_max) | |
| logging.info("Data loading complete.") | |
| # ----------------------------- | |
| # Gradio Interface | |
| # ----------------------------- | |
| # Fix the Gradio interface creation with explicit configuration | |
| with gr.Blocks(title="Typhoon Analysis Dashboard", theme=gr.themes.Default()) as demo: | |
| gr.Markdown("# Typhoon Analysis Dashboard") | |
| with gr.Tab("Overview"): | |
| gr.Markdown(""" | |
| ## Welcome to the Typhoon Analysis Dashboard | |
| This dashboard allows you to analyze typhoon data in relation to ENSO phases. | |
| ### Features: | |
| - **Track Visualization**: View typhoon tracks by time period and ENSO phase. | |
| - **Wind Analysis**: Examine wind speed vs ONI relationships. | |
| - **Pressure Analysis**: Analyze pressure vs ONI relationships. | |
| - **Longitude Analysis**: Study typhoon generation longitude vs ONI. | |
| - **Path Animation**: View animated storm tracks on a world map. | |
| - **TSNE Cluster**: Perform t-SNE clustering on storm routes. | |
| ### Data Status: | |
| - **ONI Data**: %d years loaded | |
| - **Typhoon Data**: %d records loaded | |
| - **Merged Data**: %d typhoons with ONI values | |
| """ % (len(oni_data), len(typhoon_data), len(merged_data))) | |
| with gr.Tab("Track Visualization"): | |
| with gr.Row(): | |
| start_year = gr.Number(label="Start Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| start_month = gr.Dropdown(label="Start Month", choices=list(range(1, 13)), value=1) | |
| end_year = gr.Number(label="End Year", value=2024, minimum=1900, maximum=2024, step=1) | |
| end_month = gr.Dropdown(label="End Month", choices=list(range(1, 13)), value=6) | |
| enso_phase = gr.Dropdown(label="ENSO Phase", choices=['all', 'El Nino', 'La Nina', 'Neutral'], value='all') | |
| typhoon_search = gr.Textbox(label="Typhoon Search") | |
| analyze_btn = gr.Button("Generate Tracks") | |
| tracks_plot = gr.Plot(label="Typhoon Tracks", elem_id="tracks_plot") | |
| typhoon_count = gr.Textbox(label="Number of Typhoons Displayed") | |
| analyze_btn.click(fn=get_full_tracks, | |
| inputs=[start_year, start_month, end_year, end_month, enso_phase, typhoon_search], | |
| outputs=[tracks_plot, typhoon_count]) | |
| with gr.Tab("Wind Analysis"): | |
| with gr.Row(): | |
| wind_start_year = gr.Number(label="Start Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| wind_start_month = gr.Dropdown(label="Start Month", choices=list(range(1, 13)), value=1) | |
| wind_end_year = gr.Number(label="End Year", value=2024, minimum=1900, maximum=2024, step=1) | |
| wind_end_month = gr.Dropdown(label="End Month", choices=list(range(1, 13)), value=6) | |
| wind_enso_phase = gr.Dropdown(label="ENSO Phase", choices=['all', 'El Nino', 'La Nina', 'Neutral'], value='all') | |
| wind_typhoon_search = gr.Textbox(label="Typhoon Search") | |
| wind_analyze_btn = gr.Button("Generate Wind Analysis") | |
| wind_scatter = gr.Plot(label="Wind Speed vs ONI") | |
| wind_regression_results = gr.Textbox(label="Wind Regression Results") | |
| wind_analyze_btn.click(fn=get_wind_analysis, | |
| inputs=[wind_start_year, wind_start_month, wind_end_year, wind_end_month, wind_enso_phase, wind_typhoon_search], | |
| outputs=[wind_scatter, wind_regression_results]) | |
| with gr.Tab("Pressure Analysis"): | |
| with gr.Row(): | |
| pressure_start_year = gr.Number(label="Start Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| pressure_start_month = gr.Dropdown(label="Start Month", choices=list(range(1, 13)), value=1) | |
| pressure_end_year = gr.Number(label="End Year", value=2024, minimum=1900, maximum=2024, step=1) | |
| pressure_end_month = gr.Dropdown(label="End Month", choices=list(range(1, 13)), value=6) | |
| pressure_enso_phase = gr.Dropdown(label="ENSO Phase", choices=['all', 'El Nino', 'La Nina', 'Neutral'], value='all') | |
| pressure_typhoon_search = gr.Textbox(label="Typhoon Search") | |
| pressure_analyze_btn = gr.Button("Generate Pressure Analysis") | |
| pressure_scatter = gr.Plot(label="Pressure vs ONI") | |
| pressure_regression_results = gr.Textbox(label="Pressure Regression Results") | |
| pressure_analyze_btn.click(fn=get_pressure_analysis, | |
| inputs=[pressure_start_year, pressure_start_month, pressure_end_year, pressure_end_month, pressure_enso_phase, pressure_typhoon_search], | |
| outputs=[pressure_scatter, pressure_regression_results]) | |
| with gr.Tab("Longitude Analysis"): | |
| with gr.Row(): | |
| lon_start_year = gr.Number(label="Start Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| lon_start_month = gr.Dropdown(label="Start Month", choices=list(range(1, 13)), value=1) | |
| lon_end_year = gr.Number(label="End Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| lon_end_month = gr.Dropdown(label="End Month", choices=list(range(1, 13)), value=6) | |
| lon_enso_phase = gr.Dropdown(label="ENSO Phase", choices=['all', 'El Nino', 'La Nina', 'Neutral'], value='all') | |
| lon_typhoon_search = gr.Textbox(label="Typhoon Search (Optional)") | |
| lon_analyze_btn = gr.Button("Generate Longitude Analysis") | |
| regression_plot = gr.Plot(label="Longitude vs ONI") | |
| slopes_text = gr.Textbox(label="Regression Slopes") | |
| lon_regression_results = gr.Textbox(label="Longitude Regression Results") | |
| lon_analyze_btn.click(fn=get_longitude_analysis, | |
| inputs=[lon_start_year, lon_start_month, lon_end_year, lon_end_month, lon_enso_phase, lon_typhoon_search], | |
| outputs=[regression_plot, slopes_text, lon_regression_results]) | |
| with gr.Tab("Tropical Cyclone Path Animation"): | |
| with gr.Row(): | |
| year_dropdown = gr.Dropdown(label="Year", choices=[str(y) for y in range(1950, 2025)], value="2000") | |
| basin_constant = gr.Textbox(value="All Basins", visible=False) | |
| with gr.Row(): | |
| typhoon_dropdown = gr.Dropdown(label="Tropical Cyclone") | |
| standard_dropdown = gr.Dropdown(label="Classification Standard", choices=['atlantic', 'taiwan'], value='atlantic') | |
| animate_btn = gr.Button("Generate Animation") | |
| path_video = gr.Video(label="Tropical Cyclone Path Animation", format="mp4", interactive=False, elem_id="path_video") | |
| animation_info = gr.Markdown(""" | |
| ### Animation Instructions | |
| 1. Select a year. | |
| 2. Choose a tropical cyclone from the populated list. | |
| 3. Select a classification standard (Atlantic or Taiwan). | |
| 4. Click "Generate Animation". | |
| 5. The animation displays the storm track on a world map with dynamic sidebar information. | |
| """) | |
| # Update typhoon dropdown using fixed function | |
| year_dropdown.change(fn=update_typhoon_options_fixed, | |
| inputs=[year_dropdown, basin_constant], | |
| outputs=typhoon_dropdown) | |
| animate_btn.click(fn=simplified_track_video, | |
| inputs=[year_dropdown, basin_constant, typhoon_dropdown, standard_dropdown], | |
| outputs=path_video) | |
| with gr.Tab("TSNE Cluster"): | |
| with gr.Row(): | |
| tsne_start_year = gr.Number(label="Start Year", value=2000, minimum=1900, maximum=2024, step=1) | |
| tsne_start_month = gr.Dropdown(label="Start Month", choices=list(range(1, 13)), value=1) | |
| tsne_end_year = gr.Number(label="End Year", value=2024, minimum=1900, maximum=2024, step=1) | |
| tsne_end_month = gr.Dropdown(label="End Month", choices=list(range(1, 13)), value=12) | |
| tsne_enso_phase = gr.Dropdown(label="ENSO Phase", choices=['all', 'El Nino', 'La Nina', 'Neutral'], value='all') | |
| tsne_season = gr.Dropdown(label="Season", choices=['all', 'summer', 'winter'], value='all') | |
| tsne_analyze_btn = gr.Button("Analyze") | |
| tsne_plot = gr.Plot(label="t-SNE Clusters") | |
| routes_plot = gr.Plot(label="Typhoon Routes with Mean Routes") | |
| stats_plot = gr.Plot(label="Cluster Statistics") | |
| cluster_info = gr.Textbox(label="Cluster Information", lines=10) | |
| tsne_analyze_btn.click(fn=update_route_clusters, | |
| inputs=[tsne_start_year, tsne_start_month, tsne_end_year, tsne_end_month, tsne_enso_phase, tsne_season], | |
| outputs=[tsne_plot, routes_plot, stats_plot, cluster_info]) | |
| if __name__ == "__main__": | |
| # Remove the share parameter for HuggingFace Spaces | |
| demo.launch() |