Update app.py

app.py

@@ -105,6 +105,7 @@ CACHE_EXPIRY_DAYS = 1
 # -----------------------------
 # ENHANCED: Color Maps and Standards with TD Support
 # -----------------------------
+# Enhanced color mapping with TD support (for Plotly)
 enhanced_color_map = {
     'Unknown': 'rgb(200, 200, 200)',
     'Tropical Depression': 'rgb(128, 128, 128)',  # NEW: Gray for TD
@@ -116,6 +117,43 @@ enhanced_color_map = {
     'C5 Super Typhoon': 'rgb(255, 0, 0)'
 }
 
+# Matplotlib-compatible color mapping (hex colors)
+matplotlib_color_map = {
+    'Unknown': '#C8C8C8',
+    'Tropical Depression': '#808080',  # Gray for TD
+    'Tropical Storm': '#0000FF',       # Blue
+    'C1 Typhoon': '#00FFFF',          # Cyan
+    'C2 Typhoon': '#00FF00',          # Green
+    'C3 Strong Typhoon': '#FFFF00',   # Yellow
+    'C4 Very Strong Typhoon': '#FFA500', # Orange
+    'C5 Super Typhoon': '#FF0000'     # Red
+}
+
+def rgb_string_to_hex(rgb_string):
+    """Convert 'rgb(r,g,b)' string to hex color for matplotlib"""
+    try:
+        # Extract numbers from 'rgb(r,g,b)' format
+        import re
+        numbers = re.findall(r'\d+', rgb_string)
+        if len(numbers) == 3:
+            r, g, b = map(int, numbers)
+            return f'#{r:02x}{g:02x}{b:02x}'
+        else:
+            return '#808080'  # Default gray
+    except:
+        return '#808080'  # Default gray
+
+def get_matplotlib_color(category):
+    """Get matplotlib-compatible color for a storm category"""
+    return matplotlib_color_map.get(category, '#808080')
+
+# Cluster colors for route visualization
+CLUSTER_COLORS = [
+    '#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4', '#FFEAA7',
+    '#DDA0DD', '#98D8C8', '#F7DC6F', '#BB8FCE', '#85C1E9',
+    '#F8C471', '#82E0AA', '#F1948A', '#85C1E9', '#D2B4DE'
+]
+
 # Original color map for backward compatibility
 color_map = {
     'C5 Super Typhoon': 'rgb(255, 0, 0)',
@@ -688,11 +726,11 @@ def classify_enso_phases(oni_value):
         return 'Neutral'
 
 # -----------------------------
-# NEW: Advanced ML Features
+# NEW: ADVANCED ML FEATURES WITH ROUTE VISUALIZATION
 # -----------------------------
 
 def extract_storm_features(typhoon_data):
-    """Extract features for clustering analysis"""
+    """Extract comprehensive features for clustering analysis"""
     # Group by storm ID to get storm-level features
     storm_features = typhoon_data.groupby('SID').agg({
         'USA_WIND': ['max', 'mean', 'std'],
@@ -715,6 +753,47 @@ def extract_storm_features(typhoon_data):
     genesis_data.columns = ['genesis_lat', 'genesis_lon', 'genesis_intensity']
     storm_features = storm_features.merge(genesis_data, on='SID', how='left')
     
+    # Add track shape features
+    track_stats = []
+    for sid in storm_features['SID']:
+        storm_track = typhoon_data[typhoon_data['SID'] == sid].sort_values('ISO_TIME')
+        if len(storm_track) > 2:
+            # Calculate track curvature and direction changes
+            lats = storm_track['LAT'].values
+            lons = storm_track['LON'].values
+            
+            # Calculate bearing changes
+            bearing_changes = []
+            for i in range(1, len(lats)-1):
+                # Simple bearing calculation
+                dlat1 = lats[i] - lats[i-1]
+                dlon1 = lons[i] - lons[i-1]
+                dlat2 = lats[i+1] - lats[i]
+                dlon2 = lons[i+1] - lons[i]
+                
+                angle1 = np.arctan2(dlat1, dlon1)
+                angle2 = np.arctan2(dlat2, dlon2)
+                change = abs(angle2 - angle1)
+                bearing_changes.append(change)
+            
+            avg_curvature = np.mean(bearing_changes) if bearing_changes else 0
+            total_distance = np.sum(np.sqrt((np.diff(lats)**2 + np.diff(lons)**2)))
+            
+            track_stats.append({
+                'SID': sid,
+                'avg_curvature': avg_curvature,
+                'total_distance': total_distance
+            })
+        else:
+            track_stats.append({
+                'SID': sid,
+                'avg_curvature': 0,
+                'total_distance': 0
+            })
+    
+    track_stats_df = pd.DataFrame(track_stats)
+    storm_features = storm_features.merge(track_stats_df, on='SID', how='left')
+    
     return storm_features
 
 def perform_dimensionality_reduction(storm_features, method='umap', n_components=2):
@@ -754,10 +833,10 @@ def perform_dimensionality_reduction(storm_features, method='umap', n_components
     
     return embedding, feature_cols, scaler
 
-def cluster_storms(embedding, method='dbscan'):
+def cluster_storms(embedding, method='dbscan', eps=0.5, min_samples=3):
     """Cluster storms based on their embedding"""
     if method.lower() == 'dbscan':
-        clusterer = DBSCAN(eps=0.5, min_samples=5)
+        clusterer = DBSCAN(eps=eps, min_samples=min_samples)
     elif method.lower() == 'kmeans':
         clusterer = KMeans(n_clusters=5, random_state=42)
     else:
@@ -766,8 +845,8 @@ def cluster_storms(embedding, method='dbscan'):
     clusters = clusterer.fit_predict(embedding)
     return clusters
 
-def create_clustering_visualization(storm_features, typhoon_data, method='umap'):
-    """Create interactive clustering visualization"""
+def create_advanced_clustering_visualization(storm_features, typhoon_data, method='umap', show_routes=True):
+    """Create comprehensive clustering visualization with route display"""
     try:
         # Perform dimensionality reduction
         embedding, feature_cols, scaler = perform_dimensionality_reduction(storm_features, method)
@@ -785,37 +864,179 @@ def create_clustering_visualization(storm_features, typhoon_data, method='umap')
         storm_info = typhoon_data.groupby('SID').first()[['NAME', 'SEASON']].reset_index()
         storm_features_viz = storm_features_viz.merge(storm_info, on='SID', how='left')
         
-        # Create interactive plot
-        fig = px.scatter(
-            storm_features_viz,
-            x='dim1',
-            y='dim2',
-            color='cluster',
-            hover_data=['NAME', 'SEASON', 'USA_WIND_max', 'USA_PRES_min'],
-            title=f'Storm Clustering using {method.upper()}',
-            labels={
-                'dim1': f'{method.upper()} Dimension 1',
-                'dim2': f'{method.upper()} Dimension 2',
-                'cluster': 'Cluster'
-            }
-        )
+        if show_routes:
+            # Create subplot with both scatter plot and route map
+            fig = make_subplots(
+                rows=1, cols=2,
+                subplot_titles=(
+                    f'Storm Clustering using {method.upper()}',
+                    'Clustered Storm Routes'
+                ),
+                specs=[[{"type": "scatter"}, {"type": "geo"}]],
+                column_widths=[0.5, 0.5]
+            )
+            
+            # Add clustering scatter plot
+            unique_clusters = sorted(storm_features_viz['cluster'].unique())
+            for i, cluster in enumerate(unique_clusters):
+                cluster_data = storm_features_viz[storm_features_viz['cluster'] == cluster]
+                color = CLUSTER_COLORS[i % len(CLUSTER_COLORS)] if cluster != -1 else '#CCCCCC'
+                cluster_name = f'Cluster {cluster}' if cluster != -1 else 'Noise'
+                
+                fig.add_trace(
+                    go.Scatter(
+                        x=cluster_data['dim1'],
+                        y=cluster_data['dim2'],
+                        mode='markers',
+                        marker=dict(color=color, size=8),
+                        name=cluster_name,
+                        hovertemplate=(
+                            '<b>%{customdata[0]}</b><br>'
+                            'Season: %{customdata[1]}<br>'
+                            'Max Wind: %{customdata[2]:.0f} kt<br>'
+                            'Min Pressure: %{customdata[3]:.0f} hPa<br>'
+                            'Track Length: %{customdata[4]:.0f} points<br>'
+                            '<extra></extra>'
+                        ),
+                        customdata=np.column_stack((
+                            cluster_data['NAME'],
+                            cluster_data['SEASON'],
+                            cluster_data['USA_WIND_max'],
+                            cluster_data['USA_PRES_min'],
+                            cluster_data['track_length']
+                        ))
+                    ),
+                    row=1, col=1
+                )
+            
+            # Add route map
+            for i, cluster in enumerate(unique_clusters):
+                if cluster == -1:  # Skip noise for route visualization
+                    continue
+                    
+                cluster_storms = storm_features_viz[storm_features_viz['cluster'] == cluster]['SID'].tolist()
+                color = CLUSTER_COLORS[i % len(CLUSTER_COLORS)]
+                
+                for j, sid in enumerate(cluster_storms[:10]):  # Limit to 10 storms per cluster for performance
+                    storm_track = typhoon_data[typhoon_data['SID'] == sid].sort_values('ISO_TIME')
+                    if len(storm_track) > 1:
+                        storm_name = storm_track['NAME'].iloc[0] if pd.notna(storm_track['NAME'].iloc[0]) else 'UNNAMED'
+                        
+                        fig.add_trace(
+                            go.Scattergeo(
+                                lon=storm_track['LON'],
+                                lat=storm_track['LAT'],
+                                mode='lines+markers',
+                                line=dict(color=color, width=2),
+                                marker=dict(color=color, size=4),
+                                name=f'C{cluster}: {storm_name}' if j == 0 else None,
+                                showlegend=(j == 0),
+                                hovertemplate=(
+                                    f'<b>{storm_name}</b><br>'
+                                    'Lat: %{lat:.1f}°<br>'
+                                    'Lon: %{lon:.1f}°<br>'
+                                    f'Cluster: {cluster}<br>'
+                                    '<extra></extra>'
+                                )
+                            ),
+                            row=1, col=2
+                        )
+            
+            # Update layout
+            fig.update_layout(
+                title_text="Advanced Storm Clustering Analysis with Route Visualization",
+                height=600,
+                showlegend=True
+            )
+            
+            # Update geo layout
+            fig.update_geos(
+                projection_type="natural earth",
+                showland=True,
+                landcolor="LightGray",
+                showocean=True,
+                oceancolor="LightBlue",
+                showcoastlines=True,
+                coastlinecolor="Gray",
+                center=dict(lat=20, lon=140),
+                row=1, col=2
+            )
+            
+            # Update scatter plot axes
+            fig.update_xaxes(title_text=f"{method.upper()} Dimension 1", row=1, col=1)
+            fig.update_yaxes(title_text=f"{method.upper()} Dimension 2", row=1, col=1)
+            
+        else:
+            # Simple scatter plot only
+            fig = px.scatter(
+                storm_features_viz,
+                x='dim1',
+                y='dim2',
+                color='cluster',
+                hover_data=['NAME', 'SEASON', 'USA_WIND_max', 'USA_PRES_min'],
+                title=f'Storm Clustering using {method.upper()}',
+                labels={
+                    'dim1': f'{method.upper()} Dimension 1',
+                    'dim2': f'{method.upper()} Dimension 2',
+                    'cluster': 'Cluster'
+                }
+            )
         
-        # Add cluster statistics
+        # Generate detailed cluster statistics
         cluster_stats = storm_features_viz.groupby('cluster').agg({
-            'USA_WIND_max': 'mean',
-            'USA_PRES_min': 'mean',
-            'track_length': 'mean',
+            'USA_WIND_max': ['mean', 'std', 'min', 'max'],
+            'USA_PRES_min': ['mean', 'std', 'min', 'max'],
+            'track_length': ['mean', 'std'],
+            'genesis_lat': 'mean',
+            'genesis_lon': 'mean',
+            'total_distance': 'mean',
+            'avg_curvature': 'mean',
             'SID': 'count'
         }).round(2)
         
-        stats_text = "Cluster Statistics:\n"
-        for cluster, stats in cluster_stats.iterrows():
-            if cluster != -1:  # Skip noise points in DBSCAN
-                stats_text += f"Cluster {cluster}: {stats['SID']} storms, avg max wind: {stats['USA_WIND_max']} kt\n"
+        # Flatten column names for readability
+        cluster_stats.columns = ['_'.join(col).strip() for col in cluster_stats.columns]
+        
+        stats_text = "🌀 ADVANCED CLUSTER ANALYSIS RESULTS\n" + "="*50 + "\n\n"
+        
+        for cluster in sorted(storm_features_viz['cluster'].unique()):
+            if cluster == -1:
+                stats_text += f"🔸 NOISE POINTS: {cluster_stats.loc[-1, 'SID_count']} storms\n\n"
+                continue
+            
+            cluster_row = cluster_stats.loc[cluster]
+            storm_count = int(cluster_row['SID_count'])
+            
+            stats_text += f"🌪️ CLUSTER {cluster}: {storm_count} storms\n"
+            stats_text += f"   Intensity: {cluster_row['USA_WIND_max_mean']:.1f} ± {cluster_row['USA_WIND_max_std']:.1f} kt\n"
+            stats_text += f"   Pressure: {cluster_row['USA_PRES_min_mean']:.1f} ± {cluster_row['USA_PRES_min_std']:.1f} hPa\n"
+            stats_text += f"   Track Length: {cluster_row['track_length_mean']:.1f} ± {cluster_row['track_length_std']:.1f} points\n"
+            stats_text += f"   Genesis Region: {cluster_row['genesis_lat']:.1f}°N, {cluster_row['genesis_lon']:.1f}°E\n"
+            stats_text += f"   Avg Distance: {cluster_row['total_distance_mean']:.2f} degrees\n"
+            stats_text += f"   Avg Curvature: {cluster_row['avg_curvature_mean']:.3f} radians\n\n"
+        
+        # Add feature importance summary
+        stats_text += "📊 CLUSTERING FEATURES USED:\n"
+        stats_text += f"   • Storm intensity (max/mean/std wind & pressure)\n"
+        stats_text += f"   • Track characteristics (length, curvature, distance)\n"
+        stats_text += f"   • Genesis location (lat/lon)\n"
+        stats_text += f"   • Geographic range (lat/lon span)\n"
+        stats_text += f"   • Total features: {len(feature_cols)}\n\n"
+        
+        stats_text += f"🎯 ALGORITHM: {method.upper()} + DBSCAN clustering\n"
+        stats_text += f"📈 CLUSTERS FOUND: {len([c for c in storm_features_viz['cluster'].unique() if c != -1])}\n"
         
         return fig, stats_text, storm_features_viz
+        
     except Exception as e:
-        return None, f"Error in clustering: {str(e)}", None
+        error_fig = go.Figure()
+        error_fig.add_annotation(
+            text=f"Error in clustering analysis: {str(e)}",
+            xref="paper", yref="paper",
+            x=0.5, y=0.5, xanchor='center', yanchor='middle',
+            showarrow=False, font_size=16
+        )
+        return error_fig, f"Error in clustering: {str(e)}", None
 
 # -----------------------------
 # NEW: Optional CNN Implementation
@@ -1134,33 +1355,33 @@ def get_longitude_analysis(start_year, start_month, end_year, end_month, enso_ph
     return fig, slopes_text, regression
 
 def categorize_typhoon_by_standard(wind_speed, standard='atlantic'):
-    """Categorize typhoon by standard with enhanced TD support"""
+    """Categorize typhoon by standard with enhanced TD support - FIXED for matplotlib"""
     if pd.isna(wind_speed):
         return 'Tropical Depression', '#808080'
     
     if standard=='taiwan':
         wind_speed_ms = wind_speed * 0.514444
         if wind_speed_ms >= 51.0:
-            return 'Strong Typhoon', taiwan_standard['Strong Typhoon']['hex']
+            return 'Strong Typhoon', '#FF0000'  # Red
         elif wind_speed_ms >= 33.7:
-            return 'Medium Typhoon', taiwan_standard['Medium Typhoon']['hex']
+            return 'Medium Typhoon', '#FFA500'  # Orange
         elif wind_speed_ms >= 17.2:
-            return 'Mild Typhoon', taiwan_standard['Mild Typhoon']['hex']
-        return 'Tropical Depression', taiwan_standard['Tropical Depression']['hex']
+            return 'Mild Typhoon', '#FFFF00'   # Yellow
+        return 'Tropical Depression', '#808080'  # Gray
     else:
         if wind_speed >= 137:
-            return 'C5 Super Typhoon', atlantic_standard['C5 Super Typhoon']['hex']
+            return 'C5 Super Typhoon', '#FF0000'      # Red
         elif wind_speed >= 113:
-            return 'C4 Very Strong Typhoon', atlantic_standard['C4 Very Strong Typhoon']['hex']
+            return 'C4 Very Strong Typhoon', '#FFA500' # Orange
         elif wind_speed >= 96:
-            return 'C3 Strong Typhoon', atlantic_standard['C3 Strong Typhoon']['hex']
+            return 'C3 Strong Typhoon', '#FFFF00'     # Yellow
         elif wind_speed >= 83:
-            return 'C2 Typhoon', atlantic_standard['C2 Typhoon']['hex']
+            return 'C2 Typhoon', '#00FF00'            # Green
         elif wind_speed >= 64:
-            return 'C1 Typhoon', atlantic_standard['C1 Typhoon']['hex']
+            return 'C1 Typhoon', '#00FFFF'            # Cyan
         elif wind_speed >= 34:
-            return 'Tropical Storm', atlantic_standard['Tropical Storm']['hex']
-        return 'Tropical Depression', atlantic_standard['Tropical Depression']['hex']
+            return 'Tropical Storm', '#0000FF'        # Blue
+        return 'Tropical Depression', '#808080'       # Gray
 
 # -----------------------------
 # ENHANCED: Animation Functions
@@ -1234,7 +1455,7 @@ def update_typhoon_options_enhanced(year, basin):
         return gr.update(choices=["Error loading storms"], value=None)
 
 def generate_enhanced_track_video(year, typhoon_selection, standard):
-    """Enhanced track video generation with TD support and 2025 compatibility"""
+    """Enhanced track video generation with TD support and 2025 compatibility - FIXED color handling"""
     if not typhoon_selection or typhoon_selection == "No storms found":
         return None
     
@@ -1245,6 +1466,7 @@ def generate_enhanced_track_video(year, typhoon_selection, standard):
         # Get storm data
         storm_df = typhoon_data[typhoon_data['SID'] == sid].copy()
         if storm_df.empty:
+            print(f"No data found for storm {sid}")
             return None
         
         # Sort by time
@@ -1264,6 +1486,8 @@ def generate_enhanced_track_video(year, typhoon_selection, standard):
         storm_name = storm_df['NAME'].iloc[0] if pd.notna(storm_df['NAME'].iloc[0]) else "UNNAMED"
         season = storm_df['SEASON'].iloc[0] if 'SEASON' in storm_df.columns else year
         
+        print(f"Generating video for {storm_name} ({sid}) with {len(lats)} track points")
+        
         # Create figure with enhanced map
         fig, ax = plt.subplots(figsize=(14, 8), subplot_kw={'projection': ccrs.PlateCarree()})
         
@@ -1297,48 +1521,60 @@ def generate_enhanced_track_video(year, typhoon_selection, standard):
                           fontsize=11, verticalalignment='top',
                           bbox=dict(boxstyle="round,pad=0.5", facecolor='white', alpha=0.9))
         
-        # Color legend with TD support
+        # Color legend with TD support - FIXED
         legend_elements = []
-        for category, color in enhanced_color_map.items():
-            legend_elements.append(plt.Line2D([0], [0], marker='o', color='w',
-                                            markerfacecolor=color, markersize=8, label=category))
+        for category in ['Tropical Depression', 'Tropical Storm', 'C1 Typhoon', 'C2 Typhoon', 
+                        'C3 Strong Typhoon', 'C4 Very Strong Typhoon', 'C5 Super Typhoon']:
+            if category in matplotlib_color_map:
+                color = get_matplotlib_color(category)
+                legend_elements.append(plt.Line2D([0], [0], marker='o', color='w',
+                                                markerfacecolor=color, markersize=8, label=category))
         
         ax.legend(handles=legend_elements, loc='upper right', fontsize=9)
         
         def animate(frame):
-            if frame >= len(lats):
+            try:
+                if frame >= len(lats):
+                    return line, point, info_box
+                
+                # Update track line
+                line.set_data(lons[:frame+1], lats[:frame+1])
+                
+                # Update current position
+                current_wind = winds[frame]
+                category = categorize_typhoon_enhanced(current_wind)
+                color = get_matplotlib_color(category)  # FIXED: Use matplotlib-compatible color
+                
+                # Debug print for first few frames
+                if frame < 3:
+                    print(f"Frame {frame}: Wind={current_wind:.1f}kt, Category={category}, Color={color}")
+                
+                point.set_data([lons[frame]], [lats[frame]])
+                point.set_color(color)
+                point.set_markersize(8 + current_wind/10)  # Size based on intensity
+                
+                # Enhanced info display
+                if 'ISO_TIME' in storm_df.columns and frame < len(storm_df):
+                    current_time = storm_df.iloc[frame]['ISO_TIME']
+                    time_str = current_time.strftime('%Y-%m-%d %H:%M UTC') if pd.notna(current_time) else 'Unknown'
+                else:
+                    time_str = f"Step {frame+1}"
+                
+                info_text = (
+                    f"Storm: {storm_name}\n"
+                    f"Time: {time_str}\n"
+                    f"Position: {lats[frame]:.1f}°N, {lons[frame]:.1f}°E\n"
+                    f"Max Wind: {current_wind:.0f} kt\n"
+                    f"Category: {category}\n"
+                    f"Frame: {frame+1}/{len(lats)}"
+                )
+                info_box.set_text(info_text)
+                
+                return line, point, info_box
+                
+            except Exception as e:
+                print(f"Error in animate frame {frame}: {e}")
                 return line, point, info_box
-            
-            # Update track line
-            line.set_data(lons[:frame+1], lats[:frame+1])
-            
-            # Update current position
-            current_wind = winds[frame]
-            category = categorize_typhoon_enhanced(current_wind)
-            color = enhanced_color_map[category]
-            
-            point.set_data([lons[frame]], [lats[frame]])
-            point.set_color(color)
-            point.set_markersize(8 + current_wind/10)  # Size based on intensity
-            
-            # Enhanced info display
-            if 'ISO_TIME' in storm_df.columns:
-                current_time = storm_df.iloc[frame]['ISO_TIME']
-                time_str = current_time.strftime('%Y-%m-%d %H:%M UTC') if pd.notna(current_time) else 'Unknown'
-            else:
-                time_str = f"Step {frame+1}"
-            
-            info_text = (
-                f"Storm: {storm_name}\n"
-                f"Time: {time_str}\n"
-                f"Position: {lats[frame]:.1f}°N, {lons[frame]:.1f}°E\n"
-                f"Max Wind: {current_wind:.0f} kt\n"
-                f"Category: {category}\n"
-                f"Frame: {frame+1}/{len(lats)}"
-            )
-            info_box.set_text(info_text)
-            
-            return line, point, info_box
         
         # Create animation
         anim = animation.FuncAnimation(
@@ -1356,18 +1592,22 @@ def generate_enhanced_track_video(year, typhoon_selection, standard):
             extra_args=['-pix_fmt', 'yuv420p']  # Better compatibility
         )
         
+        print(f"Saving animation to {temp_file.name}")
         anim.save(temp_file.name, writer=writer, dpi=100)
         plt.close(fig)
         
+        print(f"✅ Video generated successfully: {temp_file.name}")
         return temp_file.name
         
     except Exception as e:
-        print(f"Error generating video: {e}")
+        print(f"❌ Error generating video: {e}")
+        import traceback
+        traceback.print_exc()
         return None
 
-# Simplified wrapper for backward compatibility
+# Simplified wrapper for backward compatibility - FIXED
 def simplified_track_video(year, basin, typhoon, standard):
-    """Simplified track video function"""
+    """Simplified track video function with fixed color handling"""
     if not typhoon:
         return None
     return generate_enhanced_track_video(year, typhoon, standard)
@@ -1436,7 +1676,7 @@ def create_interface():
                 This dashboard provides comprehensive analysis of typhoon data in relation to ENSO phases with advanced machine learning capabilities.
 
                 ### 🆕 Enhanced Features:
-                - **📈 Advanced ML Clustering**: UMAP/t-SNE storm pattern analysis
+                - **📈 Advanced ML Clustering**: UMAP/t-SNE storm pattern analysis with route visualization
                 - **🤖 Optional CNN Predictions**: Deep learning intensity forecasting
                 - **🌀 Complete TD Support**: Now includes Tropical Depressions (< 34 kt)
                 - **📅 2025 Data Ready**: Real-time compatibility with current year data
@@ -1455,8 +1695,9 @@ def create_interface():
                 - **Platform Compatibility**: ✅ Optimized for Hugging Face Spaces
                 """)
 
-            with gr.Tab("🔍 Advanced ML Clustering"):
-                gr.Markdown("## Storm Pattern Analysis using UMAP/t-SNE")
+            with gr.Tab("🔍 Advanced ML Clustering with Routes"):
+                gr.Markdown("## Storm Pattern Analysis using UMAP/t-SNE with Route Visualization")
+                gr.Markdown("**This tab shows both the dimensional clustering analysis AND the actual storm tracks colored by cluster**")
                 
                 with gr.Row():
                     reduction_method = gr.Dropdown(
@@ -1464,40 +1705,50 @@ def create_interface():
                         value='UMAP' if UMAP_AVAILABLE else 't-SNE',
                         label="Dimensionality Reduction Method"
                     )
-                    cluster_method = gr.Dropdown(
-                        choices=['DBSCAN', 'K-Means'],
-                        value='DBSCAN', 
-                        label="Clustering Method"
+                    show_routes = gr.Checkbox(
+                        label="Show Storm Routes on Map",
+                        value=True,
+                        info="Display actual storm tracks colored by cluster"
                     )
                 
-                analyze_clusters_btn = gr.Button("🎯 Analyze Storm Clusters", variant="primary")
+                analyze_clusters_btn = gr.Button("🎯 Analyze Storm Clusters & Routes", variant="primary")
                 
                 with gr.Row():
-                    cluster_plot = gr.Plot(label="Storm Clustering Visualization")
-                    cluster_stats = gr.Textbox(label="Cluster Statistics", lines=10)
+                    cluster_plot = gr.Plot(label="Storm Clustering with Route Visualization", height=600)
                 
-                def run_clustering_analysis(method):
+                with gr.Row():
+                    cluster_stats = gr.Textbox(label="Detailed Cluster Statistics", lines=15, max_lines=20)
+                
+                def run_advanced_clustering_analysis(method, show_routes):
                     try:
                         # Extract features for clustering
                         storm_features = extract_storm_features(typhoon_data)
-                        fig, stats, _ = create_clustering_visualization(storm_features, typhoon_data, method.lower())
+                        fig, stats, _ = create_advanced_clustering_visualization(storm_features, typhoon_data, method.lower(), show_routes)
                         return fig, stats
                     except Exception as e:
-                        return None, f"Error: {str(e)}"
+                        import traceback
+                        error_details = traceback.format_exc()
+                        return None, f"Error: {str(e)}\n\nDetails:\n{error_details}"
                 
                 analyze_clusters_btn.click(
-                    fn=run_clustering_analysis,
-                    inputs=[reduction_method],
+                    fn=run_advanced_clustering_analysis,
+                    inputs=[reduction_method, show_routes],
                     outputs=[cluster_plot, cluster_stats]
                 )
                 
                 gr.Markdown("""
-                ### ℹ️ About Storm Clustering:
-                - **UMAP**: Faster and preserves global structure better
-                - **t-SNE**: Good for local neighborhood preservation
-                - **PCA**: Linear dimensionality reduction (fallback)
-                - **DBSCAN**: Density-based clustering, finds natural groupings
-                - **K-Means**: Partitions storms into K predefined clusters
+                ### 🧠 Advanced Clustering Features:
+                - **🎯 Multi-dimensional Analysis**: Uses 15+ storm characteristics including intensity, track shape, genesis location
+                - **🗺️ Route Visualization**: Shows actual storm tracks colored by cluster membership
+                - **📊 DBSCAN Clustering**: Automatically finds natural groupings without predefined cluster count
+                - **📈 Comprehensive Stats**: Detailed cluster analysis including intensity, pressure, track length, curvature
+                - **🔄 Interactive**: Hover over points to see storm details, zoom and pan the route map
+                
+                ### 📖 How to Interpret:
+                - **Left Plot**: Each dot is a storm positioned by similarity (close = similar characteristics)
+                - **Right Plot**: Actual geographic storm tracks, colored by which cluster they belong to
+                - **Cluster Colors**: Each cluster gets a unique color to identify similar storm patterns
+                - **Noise Points**: Gray points represent storms that don't fit clear patterns
                 """)
 
             with gr.Tab("🤖 Intensity Prediction"):
@@ -1666,6 +1917,125 @@ def create_interface():
                 - **⚡ Optimized Export**: Fast rendering with web-compatible video formats
                 """)
 
+            with gr.Tab("📊 Data Statistics & Insights"):
+                gr.Markdown("## Comprehensive Dataset Analysis")
+                
+                # Create enhanced data summary
+                try:
+                    if len(typhoon_data) > 0:
+                        # Storm category distribution
+                        storm_cats = typhoon_data.groupby('SID')['USA_WIND'].max().apply(categorize_typhoon_enhanced)
+                        cat_counts = storm_cats.value_counts()
+                        
+                        # Create distribution chart with enhanced colors
+                        fig_dist = px.bar(
+                            x=cat_counts.index,
+                            y=cat_counts.values,
+                            title="Storm Intensity Distribution (Including Tropical Depressions)",
+                            labels={'x': 'Category', 'y': 'Number of Storms'},
+                            color=cat_counts.index,
+                            color_discrete_map=enhanced_color_map
+                        )
+                        
+                        # Seasonal distribution
+                        if 'ISO_TIME' in typhoon_data.columns:
+                            seasonal_data = typhoon_data.copy()
+                            seasonal_data['Month'] = seasonal_data['ISO_TIME'].dt.month
+                            monthly_counts = seasonal_data.groupby(['Month', 'SID']).size().groupby('Month').size()
+                            
+                            fig_seasonal = px.bar(
+                                x=monthly_counts.index,
+                                y=monthly_counts.values,
+                                title="Seasonal Storm Distribution",
+                                labels={'x': 'Month', 'y': 'Number of Storms'},
+                                color=monthly_counts.values,
+                                color_continuous_scale='Viridis'
+                            )
+                        else:
+                            fig_seasonal = None
+                        
+                        # Basin distribution
+                        if 'SID' in typhoon_data.columns:
+                            basin_data = typhoon_data['SID'].str[:2].value_counts()
+                            fig_basin = px.pie(
+                                values=basin_data.values,
+                                names=basin_data.index,
+                                title="Distribution by Basin"
+                            )
+                        else:
+                            fig_basin = None
+                        
+                        with gr.Row():
+                            gr.Plot(value=fig_dist)
+                        
+                        if fig_seasonal:
+                            with gr.Row():
+                                gr.Plot(value=fig_seasonal)
+                        
+                        if fig_basin:
+                            with gr.Row():
+                                gr.Plot(value=fig_basin)
+                                
+                except Exception as e:
+                    gr.Markdown(f"Visualization error: {str(e)}")
+                
+                # Enhanced statistics
+                total_storms = len(typhoon_data['SID'].unique()) if 'SID' in typhoon_data.columns else 0
+                total_records = len(typhoon_data)
+                
+                if 'SEASON' in typhoon_data.columns:
+                    year_range = f"{typhoon_data['SEASON'].min():.0f}-{typhoon_data['SEASON'].max():.0f}"
+                    years_covered = typhoon_data['SEASON'].nunique()
+                else:
+                    year_range = "Unknown"
+                    years_covered = 0
+                
+                if 'SID' in typhoon_data.columns:
+                    basins_available = ', '.join(sorted(typhoon_data['SID'].str[:2].unique()))
+                    avg_storms_per_year = total_storms / max(years_covered, 1)
+                else:
+                    basins_available = "Unknown"
+                    avg_storms_per_year = 0
+                
+                # TD specific statistics
+                if 'USA_WIND' in typhoon_data.columns:
+                    td_storms = len(typhoon_data[typhoon_data['USA_WIND'] < 34]['SID'].unique())
+                    ts_storms = len(typhoon_data[(typhoon_data['USA_WIND'] >= 34) & (typhoon_data['USA_WIND'] < 64)]['SID'].unique())
+                    typhoon_storms = len(typhoon_data[typhoon_data['USA_WIND'] >= 64]['SID'].unique())
+                    td_percentage = (td_storms / max(total_storms, 1)) * 100
+                else:
+                    td_storms = ts_storms = typhoon_storms = 0
+                    td_percentage = 0
+                
+                gr.Markdown(f"""
+                ### 📈 Enhanced Dataset Summary:
+                - **Total Unique Storms**: {total_storms:,}
+                - **Total Track Records**: {total_records:,}  
+                - **Year Range**: {year_range} ({years_covered} years)
+                - **Basins Available**: {basins_available}
+                - **Average Storms/Year**: {avg_storms_per_year:.1f}
+                
+                ### 🌀 Storm Category Breakdown:
+                - **Tropical Depressions**: {td_storms:,} storms ({td_percentage:.1f}%)
+                - **Tropical Storms**: {ts_storms:,} storms
+                - **Typhoons (C1-C5)**: {typhoon_storms:,} storms
+                
+                ### 🆕 New Platform Capabilities:
+                - ✅ **Complete TD Analysis** - First platform to include comprehensive TD tracking
+                - ✅ **Advanced ML Clustering** - DBSCAN pattern recognition with route visualization  
+                - ✅ **Real-time Predictions** - Physics-based and optional CNN intensity forecasting
+                - ✅ **2025 Data Ready** - Full compatibility with current season data
+                - ✅ **Enhanced Animations** - Professional-quality storm track videos
+                - ✅ **Multi-basin Analysis** - Comprehensive Pacific and Atlantic coverage
+                
+                ### 🔬 Research Applications:
+                - Climate change impact studies
+                - Seasonal forecasting research
+                - Storm pattern classification
+                - ENSO-typhoon relationship analysis
+                - Intensity prediction model development
+                """)
+
         return demo
     except Exception as e:
         logging.error(f"Error creating Gradio interface: {e}")
@@ -1673,8 +2043,45 @@ def create_interface():
         with gr.Blocks() as demo:
             gr.Markdown("# 🌪️ Enhanced Typhoon Analysis Platform")
             gr.Markdown("**Error**: Could not load full interface. Please check logs.")
+            gr.Markdown(f"Error details: {str(e)}")
         return demo
 
+# -----------------------------
+# Color Test Functions (Optional)
+# -----------------------------
+
+def test_color_conversion():
+    """Test color conversion functions"""
+    print("🎨 Testing color conversion...")
+    
+    # Test all categories
+    test_winds = [25, 40, 70, 85, 100, 120, 150]  # TD, TS, C1, C2, C3, C4, C5
+    
+    for wind in test_winds:
+        category = categorize_typhoon_enhanced(wind)
+        plotly_color = enhanced_color_map.get(category, 'rgb(128,128,128)')
+        matplotlib_color = get_matplotlib_color(category)
+        
+        print(f"Wind: {wind:3d}kt → {category:20s} → Plotly: {plotly_color:15s} → Matplotlib: {matplotlib_color}")
+    
+    print("✅ Color conversion test complete!")
+
+def test_rgb_conversion():
+    """Test RGB string to hex conversion"""
+    test_colors = [
+        'rgb(128, 128, 128)',
+        'rgb(255, 0, 0)',
+        'rgb(0, 255, 0)',
+        'rgb(0, 0, 255)'
+    ]
+    
+    print("🔧 Testing RGB to hex conversion...")
+    for rgb_str in test_colors:
+        hex_color = rgb_string_to_hex(rgb_str)
+        print(f"{rgb_str:20s} → {hex_color}")
+    
+    print("✅ RGB conversion test complete!")
+
 # Create and launch the interface
 demo = create_interface()