Spaces:
Sleeping
Sleeping
| # -*- coding: utf-8 -*- | |
| # | |
| # PROJECT: CognitiveEDA - The AI-Augmented Data Discovery Platform | |
| # | |
| # DESCRIPTION: An enterprise-grade Gradio application that revolutionizes Exploratory | |
| # Data Analysis (EDA). By integrating Google's Gemini Pro LLM, this | |
| # tool transcends traditional data profiling to deliver a rich, | |
| # narrative-driven analysis, actionable insights, and strategic | |
| # recommendations in a single, streamlined workflow. | |
| # | |
| # SETUP: This application has external dependencies. Before running, install | |
| # all required packages using the requirements.txt file: | |
| # $ pip install -r requirements.txt | |
| # | |
| # ARCHITECTURE: The application is built upon a robust, object-oriented foundation. | |
| # - DataAnalyzer (Core Engine): An encapsulated class that holds the | |
| # DataFrame state and performs all statistical calculations and | |
| # metadata extraction efficiently, ensuring data is processed once. | |
| # - AI Integration: A dedicated module communicates with the Gemini API, | |
| # using a sophisticated, structured prompt to ensure consistent, | |
| # high-quality analytical narratives. | |
| # - Gradio Interface (UI Layer): A multi-tabbed, interactive dashboard | |
| # that logically separates the AI narrative, data profiling, static | |
| # visuals, and interactive exploration tools. | |
| # | |
| # AUTHOR: An MCP Expert in Data & AI Solutions | |
| # VERSION: 3.1 (Enterprise Edition) | |
| # LAST-UPDATE: 2023-10-28 (Added dependency check & requirements file) | |
| from __future__ import annotations | |
| import warnings | |
| import logging | |
| import os | |
| import sys | |
| import importlib.util | |
| from datetime import datetime | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import gradio as gr | |
| import numpy as np | |
| import pandas as pd | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| import google.generativeai as genai | |
| # --- Configuration & Constants --- | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - [%(levelname)s] - (%(filename)s:%(lineno)d) - %(message)s' | |
| ) | |
| warnings.filterwarnings('ignore', category=FutureWarning) | |
| class Config: | |
| """Application-wide configuration settings.""" | |
| APP_TITLE = "🚀 CognitiveEDA: AI-Augmented Data Discovery Platform" | |
| GEMINI_MODEL = 'gemini-1.5-flash-latest' | |
| CORR_THRESHOLD = 0.75 # Threshold for highlighting high correlation | |
| TOP_N_CATEGORIES = 10 # For bar charts of categorical features | |
| # --- Core Analysis Engine --- | |
| class DataAnalyzer: | |
| """ | |
| Encapsulates all data analysis logic, acting as the single source of truth | |
| for the uploaded dataset and its derived metadata. | |
| """ | |
| def __init__(self, df: pd.DataFrame): | |
| if not isinstance(df, pd.DataFrame): | |
| raise TypeError("Input must be a pandas DataFrame.") | |
| self.df = df | |
| self._metadata: Optional[Dict[str, Any]] = None | |
| logging.info(f"DataAnalyzer instantiated with DataFrame of shape: {self.df.shape}") | |
| def metadata(self) -> Dict[str, Any]: | |
| """Lazy-loads and caches comprehensive dataset metadata for efficient reuse.""" | |
| if self._metadata is None: | |
| logging.info("First access to metadata, performing extraction...") | |
| self._metadata = self._extract_metadata() | |
| return self._metadata | |
| def _extract_metadata(self) -> Dict[str, Any]: | |
| """Performs a deep scan of the DataFrame to extract key characteristics.""" | |
| rows, cols = self.df.shape | |
| numeric_cols = self.df.select_dtypes(include=np.number).columns.tolist() | |
| categorical_cols = self.df.select_dtypes(include=['object', 'category']).columns.tolist() | |
| high_corr_pairs = [] | |
| if len(numeric_cols) > 1: | |
| corr_matrix = self.df[numeric_cols].corr().abs() | |
| upper_tri = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool)) | |
| high_corr_series = upper_tri.stack() | |
| high_corr_pairs = ( | |
| high_corr_series[high_corr_series > Config.CORR_THRESHOLD] | |
| .reset_index() | |
| .rename(columns={'level_0': 'Feature 1', 'level_1': 'Feature 2', 0: 'Correlation'}) | |
| .to_dict('records') | |
| ) | |
| return { | |
| 'shape': (rows, cols), | |
| 'columns': self.df.columns.tolist(), | |
| 'numeric_cols': numeric_cols, | |
| 'categorical_cols': categorical_cols, | |
| 'memory_usage_mb': f"{self.df.memory_usage(deep=True).sum() / 1e6:.2f}", | |
| 'total_missing': int(self.df.isnull().sum().sum()), | |
| 'data_quality_score': round((self.df.notna().sum().sum() / self.df.size) * 100, 2), | |
| 'high_corr_pairs': high_corr_pairs, | |
| } | |
| def get_profiling_tables(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: | |
| """Generates structured DataFrames for data profiling.""" | |
| logging.info("Generating profiling tables for missing, numeric, and categorical data.") | |
| missing = self.df.isnull().sum() | |
| missing_df = pd.DataFrame({ | |
| 'Missing Count': missing, | |
| 'Missing Percentage (%)': (missing / len(self.df) * 100).round(2) | |
| }).reset_index().rename(columns={'index': 'Column'}).sort_values('Missing Count', ascending=False) | |
| numeric_stats = self.df[self.metadata['numeric_cols']].describe(percentiles=[.01, .25, .5, .75, .99]).T | |
| numeric_stats_df = numeric_stats.round(3).reset_index().rename(columns={'index': 'Column'}) | |
| cat_stats = self.df[self.metadata['categorical_cols']].describe(include=['object', 'category']).T | |
| cat_stats_df = cat_stats.reset_index().rename(columns={'index': 'Column'}) | |
| return missing_df, numeric_stats_df, cat_stats_df | |
| def get_overview_visuals(self) -> Tuple[go.Figure, go.Figure, go.Figure]: | |
| """Creates a set of key visualizations for a high-level overview.""" | |
| logging.info("Generating overview visualizations (types, missing data, correlation).") | |
| meta = self.metadata | |
| dtype_counts = self.df.dtypes.astype(str).value_counts() | |
| fig_types = px.pie( | |
| values=dtype_counts.values, names=dtype_counts.index, | |
| title="<b>📊 Data Type Composition</b>", hole=0.4, | |
| color_discrete_sequence=px.colors.qualitative.Pastel | |
| ) | |
| fig_types.update_traces(textposition='outside', textinfo='percent+label') | |
| missing_df = self.df.isnull().sum().reset_index(name='count').query('count > 0') | |
| fig_missing = px.bar( | |
| missing_df, x='index', y='count', title="<b>🕳️ Missing Values Distribution</b>", | |
| labels={'index': 'Column Name', 'count': 'Number of Missing Values'}, | |
| ).update_xaxes(categoryorder="total descending") | |
| fig_corr = go.Figure() | |
| if len(meta['numeric_cols']) > 1: | |
| corr_matrix = self.df[meta['numeric_cols']].corr() | |
| fig_corr = px.imshow( | |
| corr_matrix, text_auto=".2f", aspect="auto", | |
| title=f"<b>🔗 Correlation Matrix (Threshold > {Config.CORR_THRESHOLD})</b>", | |
| color_continuous_scale='RdBu_r', zmin=-1, zmax=1 | |
| ) | |
| else: | |
| fig_corr.update_layout(title="<b>🔗 Correlation Matrix (Insufficient Numeric Data)</b>") | |
| return fig_types, fig_missing, fig_corr | |
| def generate_ai_narrative(self, api_key: str) -> str: | |
| """Orchestrates the generation of the full AI-driven report using Gemini.""" | |
| logging.info("Generating AI narrative with the Gemini API.") | |
| meta = self.metadata | |
| # NOTE: The .to_markdown() method requires the 'tabulate' library. | |
| # This is handled by the pre-flight check in if __name__ == "__main__": | |
| data_snippet_md = self.df.head(5).to_markdown(index=False) | |
| prompt = f""" | |
| As "Cognitive Analyst," an elite AI data scientist, your task is to generate a comprehensive, multi-part data discovery report. | |
| Analyze the following dataset context and produce a professional, insightful, and clear analysis in Markdown format. | |
| **DATASET CONTEXT:** | |
| - **Shape:** {meta['shape'][0]} rows, {meta['shape'][1]} columns. | |
| - **Column Schema:** | |
| - Numeric: {', '.join(meta['numeric_cols']) if meta['numeric_cols'] else 'None'} | |
| - Categorical: {', '.join(meta['categorical_cols']) if meta['categorical_cols'] else 'None'} | |
| - **Data Quality Score:** {meta['data_quality_score']}% (Percentage of non-missing cells) | |
| - **Total Missing Values:** {meta['total_missing']:,} | |
| - **High-Correlation Pairs (>{Config.CORR_THRESHOLD}):** {meta['high_corr_pairs'] if meta['high_corr_pairs'] else 'None detected.'} | |
| - **Data Snippet (First 5 Rows):** | |
| {data_snippet_md} | |
| **REQUIRED REPORT STRUCTURE (Strictly use this Markdown format):** | |
| # 🚀 AI Data Discovery Report | |
| ## 📄 1. Executive Summary | |
| * **Primary Objective:** (Deduce the most likely purpose of this dataset. What problem is it trying to solve?) | |
| * **Key Finding:** (State the single most interesting or impactful insight you've discovered.) | |
| * **Overall State:** (Briefly comment on the data's quality and readiness for analysis.) | |
| ## 🧐 2. Data Profile & Quality Assessment | |
| * **First Impression:** (Describe the dataset's structure, size, and composition.) | |
| * **Data Quality Audit:** (Elaborate on the **{meta['data_quality_score']}%** quality score. Are the **{meta['total_missing']}** missing values concentrated in specific columns? Is this a major concern?) | |
| * **Redundancy Check:** (Comment on the detected high-correlation pairs. Is there a risk of multicollinearity in modeling?) | |
| ## 💡 3. Key Insights & Potential Stories | |
| * **Insight 1 (e.g., Anomaly Detected 🕵️):** (Describe a surprising pattern, outlier, or distribution in a key numeric column.) | |
| * **Insight 2 (e.g., Categorical Trend 📊):** (Analyze a key categorical column. What does its distribution reveal? Is there a dominant category?) | |
| * **Insight 3 (e.g., Relationship Hint 🔗):** (Speculate on a potential relationship between two or more columns, even if not highly correlated.) | |
| ## 🛠️ 4. Actionable Recommendations | |
| * **Data Cleaning:** | |
| - **Step 1:** (Provide a specific recommendation for handling missing data, e.g., "For `column_name`, with X% missing, consider imputation using the median due to its skewed distribution.") | |
| - **Step 2:** (Suggest actions for correlated features, e.g., "Consider dropping `Feature A` or using dimensionality reduction (PCA) due to its high correlation with `Feature B`.") | |
| * **Feature Engineering:** | |
| - **Idea 1:** (Suggest creating a new feature, e.g., "Combine `year` and `month` into a `date` feature for time-series analysis.") | |
| * **Next Analytical Steps:** | |
| - **Hypothesis to Test:** (Propose a business or research question to investigate further, e.g., "Does `customer_segment` significantly impact `total_spend`?") | |
| - **Modeling Potential:** (Suggest a suitable machine learning model, e.g., "This dataset is well-suited for a classification model to predict `is_churn`.") | |
| """ | |
| try: | |
| genai.configure(api_key=api_key) | |
| model = genai.GenerativeModel(Config.GEMINI_MODEL) | |
| response = model.generate_content(prompt) | |
| return response.text | |
| except Exception as e: | |
| logging.error(f"Gemini API call failed: {e}", exc_info=True) | |
| error_message = ( | |
| "❌ **AI Report Generation Failed**\n\n" | |
| f"**Error Details:** `{str(e)}`\n\n" | |
| "**Troubleshooting Steps:**\n" | |
| "1. Verify that your Google Gemini API key is correct and active.\n" | |
| "2. Check your network connection and firewall settings.\n" | |
| "3. Ensure the Gemini API is not experiencing an outage." | |
| ) | |
| return error_message | |
| # --- Gradio UI & Event Handlers --- | |
| def create_ui(): | |
| """Defines and builds the Gradio user interface.""" | |
| def create_histogram(analyzer: DataAnalyzer, col: str) -> go.Figure: | |
| if not col or not analyzer: return go.Figure() | |
| return px.histogram(analyzer.df, x=col, title=f"<b>Distribution of {col}</b>", marginal="box", template="plotly_white") | |
| def create_scatterplot(analyzer: DataAnalyzer, x_col: str, y_col:str, color_col:str) -> go.Figure: | |
| if not all([analyzer, x_col, y_col]): return go.Figure() | |
| return px.scatter( | |
| analyzer.df, x=x_col, y=y_col, color=color_col, | |
| title=f"<b>Scatter Plot: {x_col} vs. {y_col}</b>", template="plotly_white", | |
| color_continuous_scale=px.colors.sequential.Viridis | |
| ) | |
| def analyze_single_column(analyzer: DataAnalyzer, col: str) -> Tuple[str, go.Figure]: | |
| if not col or not analyzer: return "", go.Figure() | |
| series = analyzer.df[col] | |
| stats_md = f"### 🔎 **Deep Dive: `{col}`**\n" | |
| stats_md += f"- **Data Type:** `{series.dtype}`\n" | |
| stats_md += f"- **Unique Values:** `{series.nunique()}`\n" | |
| stats_md += f"- **Missing:** `{series.isnull().sum()}` ({series.isnull().mean():.2%})\n" | |
| fig = go.Figure() | |
| if pd.api.types.is_numeric_dtype(series): | |
| stats_md += f"- **Mean:** `{series.mean():.3f}` | **Std Dev:** `{series.std():.3f}`\n" | |
| stats_md += f"- **Median:** `{series.median():.3f}` | **Min:** `{series.min():.3f}` | **Max:** `{series.max():.3f}`\n" | |
| fig = create_histogram(analyzer, col) | |
| else: | |
| top_n = series.value_counts().nlargest(Config.TOP_N_CATEGORIES) | |
| stats_md += f"- **Top Value:** `{top_n.index[0]}` ({top_n.iloc[0]} occurrences)\n" | |
| fig = px.bar( | |
| top_n, y=top_n.index, x=top_n.values, orientation='h', | |
| title=f"<b>Top {Config.TOP_N_CATEGORIES} Categories in `{col}`</b>", | |
| labels={'y': col, 'x': 'Count'}, template="plotly_white" | |
| ).update_yaxes(categoryorder="total ascending") | |
| return stats_md, fig | |
| with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="cyan"), title=Config.APP_TITLE) as demo: | |
| state_analyzer = gr.State() | |
| gr.Markdown(f"<h1>{Config.APP_TITLE}</h1>") | |
| gr.Markdown("Upload a CSV file, provide your Gemini API key, and receive an instant, AI-driven analysis of your data.") | |
| with gr.Row(): | |
| with gr.Column(scale=3): | |
| upload_button = gr.File(label="1. Upload CSV File", file_types=[".csv"]) | |
| with gr.Column(scale=2): | |
| api_key_input = gr.Textbox(label="2. Enter Google Gemini API Key", type="password") | |
| with gr.Column(scale=1, min_width=150): | |
| analyze_button = gr.Button("✨ Generate Analysis", variant="primary") | |
| with gr.Tabs(): | |
| with gr.Tab("🤖 AI Narrative"): | |
| ai_report_output = gr.Markdown("Your AI-generated report will appear here once analysis is complete...") | |
| download_report_button = gr.Button("⬇️ Download Full Report", visible=False) | |
| with gr.Tab("Profile"): | |
| gr.Markdown("### **Detailed Data Profile**") | |
| profile_missing_df = gr.DataFrame(interactive=False, label="Missing Values") | |
| profile_numeric_df = gr.DataFrame(interactive=False, label="Numeric Stats") | |
| profile_categorical_df = gr.DataFrame(interactive=False, label="Categorical Stats") | |
| with gr.Tab("📈 Overview Visuals"): | |
| gr.Markdown("### **At-a-Glance Visualizations**") | |
| with gr.Row(): | |
| plot_types = gr.Plot() | |
| plot_missing = gr.Plot() | |
| plot_correlation = gr.Plot() | |
| with gr.Tab("🎨 Interactive Explorer"): | |
| gr.Markdown("### **Visually Explore Feature Relationships**") | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1): | |
| gr.Markdown("#### Univariate Analysis") | |
| dd_hist_col = gr.Dropdown(label="Select Column for Histogram", visible=False) | |
| with gr.Column(scale=2): | |
| plot_histogram = gr.Plot() | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1): | |
| gr.Markdown("#### Bivariate Analysis (Scatter Plot)") | |
| dd_scatter_x = gr.Dropdown(label="X-Axis (Numeric)", visible=False) | |
| dd_scatter_y = gr.Dropdown(label="Y-Axis (Numeric)", visible=False) | |
| dd_scatter_color = gr.Dropdown(label="Color By (Optional)", visible=False) | |
| with gr.Column(scale=2): | |
| plot_scatter = gr.Plot() | |
| with gr.Tab("🔍 Column Deep-Dive"): | |
| gr.Markdown("### **Inspect a Single Column in Detail**") | |
| dd_drilldown_col = gr.Dropdown(label="Select Column to Analyze", visible=False) | |
| with gr.Row(): | |
| md_drilldown_stats = gr.Markdown() | |
| plot_drilldown = gr.Plot() | |
| gr.HTML(""" | |
| <div style="text-align: center; margin-top: 20px; font-family: sans-serif; color: #777;"> | |
| <p>💡 Need an API key? Get one from <a href="https://aistudio.google.com/app/apikey" target="_blank">Google AI Studio</a>.</p> | |
| <p>CognitiveEDA v3.1 | An MCP Expert System</p> | |
| </div> | |
| """) | |
| outputs_for_main_analysis = [ | |
| state_analyzer, ai_report_output, download_report_button, | |
| profile_missing_df, profile_numeric_df, profile_categorical_df, | |
| plot_types, plot_missing, plot_correlation, | |
| dd_hist_col, dd_scatter_x, dd_scatter_y, dd_scatter_color, dd_drilldown_col | |
| ] | |
| analyze_button.click(fn=run_full_analysis, inputs=[upload_button, api_key_input], outputs=outputs_for_main_analysis) | |
| dd_hist_col.change(fn=create_histogram, inputs=[state_analyzer, dd_hist_col], outputs=plot_histogram) | |
| scatter_inputs = [state_analyzer, dd_scatter_x, dd_scatter_y, dd_scatter_color] | |
| for dd in [dd_scatter_x, dd_scatter_y, dd_scatter_color]: | |
| dd.change(fn=create_scatterplot, inputs=scatter_inputs, outputs=plot_scatter) | |
| dd_drilldown_col.change(fn=analyze_single_column, inputs=[state_analyzer, dd_drilldown_col], outputs=[md_drilldown_stats, plot_drilldown]) | |
| download_report_button.click(fn=download_report_file, inputs=[state_analyzer, ai_report_output], outputs=gr.File(label="Download Report")) | |
| return demo | |
| # --- Main Application Logic --- | |
| def run_full_analysis(file_obj: gr.File, api_key: str) -> Dict[gr.component, Any]: | |
| """Orchestrates the entire analysis pipeline upon button click.""" | |
| if file_obj is None: | |
| raise gr.Error("CRITICAL: No file uploaded. Please select a CSV file.") | |
| if not api_key: | |
| raise gr.Error("CRITICAL: Gemini API key is missing. Please provide your key.") | |
| try: | |
| logging.info(f"Processing uploaded file: {file_obj.name}") | |
| df = pd.read_csv(file_obj.name) | |
| analyzer = DataAnalyzer(df) | |
| ai_report = analyzer.generate_ai_narrative(api_key) | |
| missing_df, num_df, cat_df = analyzer.get_profiling_tables() | |
| fig_types, fig_missing, fig_corr = analyzer.get_overview_visuals() | |
| meta = analyzer.metadata | |
| all_cols, num_cols = meta['columns'], meta['numeric_cols'] | |
| return { | |
| state_analyzer: analyzer, ai_report_output: ai_report, | |
| download_report_button: gr.Button(visible=True), | |
| profile_missing_df: missing_df, profile_numeric_df: num_df, | |
| profile_categorical_df: cat_df, plot_types: fig_types, | |
| plot_missing: fig_missing, plot_correlation: fig_corr, | |
| dd_hist_col: gr.Dropdown(choices=num_cols, label="Select Numeric Column", visible=True), | |
| dd_scatter_x: gr.Dropdown(choices=num_cols, label="X-Axis (Numeric)", visible=True), | |
| dd_scatter_y: gr.Dropdown(choices=num_cols, label="Y-Axis (Numeric)", visible=True), | |
| dd_scatter_color: gr.Dropdown(choices=all_cols, label="Color By (Optional)", visible=True), | |
| dd_drilldown_col: gr.Dropdown(choices=all_cols, label="Select Column to Analyze", visible=True) | |
| } | |
| except Exception as e: | |
| logging.error(f"A critical error occurred during file processing: {e}", exc_info=True) | |
| raise gr.Error(f"Analysis Failed! The process stopped due to: {str(e)}") | |
| def download_report_file(analyzer: DataAnalyzer, ai_report_text: str) -> Optional[str]: | |
| """Generates a comprehensive Markdown file for download.""" | |
| if not analyzer: | |
| logging.warning("Download attempted without a valid analyzer object.") | |
| return None | |
| filename = f"CognitiveEDA_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.md" | |
| meta = analyzer.metadata | |
| full_report = f"# CognitiveEDA - Data Discovery Report\n" | |
| full_report += f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n" | |
| full_report += f"## Dataset Overview\n" | |
| full_report += f"- **Shape:** {meta['shape'][0]} rows x {meta['shape'][1]} columns\n" | |
| full_report += f"- **Memory Footprint:** {meta['memory_usage_mb']} MB\n" | |
| full_report += f"- **Data Quality Score:** {meta['data_quality_score']}%\n\n" | |
| full_report += "---\n\n" | |
| full_report += ai_report_text | |
| with open(filename, "w", encoding="utf-8") as f: | |
| f.write(full_report) | |
| logging.info(f"Report file generated successfully: {filename}") | |
| return filename | |
| def perform_pre_flight_checks(): | |
| """Checks for critical dependencies before launching the app.""" | |
| logging.info("Performing pre-flight dependency checks...") | |
| required_packages = ["pandas", "gradio", "plotly", "google.generativeai", "tabulate"] | |
| missing_packages = [pkg for pkg in required_packages if importlib.util.find_spec(pkg) is None] | |
| if missing_packages: | |
| logging.critical(f"Missing critical packages: {', '.join(missing_packages)}") | |
| print("\n" + "="*80) | |
| print("ERROR: Your environment is missing critical dependencies.") | |
| print(f"Missing package(s): {', '.join(missing_packages)}") | |
| print("Please install all required packages using the requirements.txt file:") | |
| print("pip install -r requirements.txt") | |
| print("="*80 + "\n") | |
| sys.exit(1) | |
| logging.info("All dependencies are satisfied. Proceeding with launch.") | |
| if __name__ == "__main__": | |
| perform_pre_flight_checks() | |
| app_instance = create_ui() | |
| app_instance.launch(debug=True, server_name="0.0.0.0") |