app.py

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import gradio as gr
import io
import base64
import tempfile
import os
from datetime import datetime

# --- Matplotlib Plot to Base64 ---
def fig_to_base64(fig):
    """Converts a Matplotlib figure to a base64 encoded PNG string."""
    buf = io.BytesIO()
    fig.savefig(buf, format='png', bbox_inches='tight')
    plt.close(fig) # Close the figure to free memory
    buf.seek(0)
    img_str = base64.b64encode(buf.read()).decode('utf-8')
    return f"data:image/png;base64,{img_str}"

# --- EDA Helper Functions (Adapted from Colab) ---

def get_initial_inspection_html(df):
    """Generates HTML for initial data inspection."""
    html = "<h2>1. Initial Data Inspection</h2>"
    # Head
    html += "<h3>(a) First 5 Rows (Head):</h3>"
    html += df.head().to_html(classes='table table-striped', border=1)
    # Tail
    html += "<h3>(b) Last 5 Rows (Tail):</h3>"
    html += df.tail().to_html(classes='table table-striped', border=1)
    # Shape
    html += "<h3>(c) Dataset Shape:</h3>"
    html += f"<p>Number of Rows: {df.shape[0]}</p>"
    html += f"<p>Number of Columns: {df.shape[1]}</p>"
    # Info
    html += "<h3>(d) Data Types and Non-Null Counts (Info):</h3>"
    buffer = io.StringIO()
    df.info(buf=buffer)
    info_str = buffer.getvalue()
    html += f"<pre>{info_str}</pre>"
    # Column Names
    html += "<h3>(e) Column Names:</h3>"
    html += f"<p><code>{list(df.columns)}</code></p>"
    return html

def get_descriptive_stats_html(df):
    """Generates HTML for descriptive statistics."""
    html = "<h2>2. Descriptive Statistics</h2>"
    # Numerical
    html += "<h3>(a) Numerical Columns Statistics:</h3>"
    try:
        num_stats = df.describe(include=np.number)
        if not num_stats.empty:
             html += num_stats.to_html(classes='table table-striped', border=1, float_format='%.2f')
        else:
            html += "<p>No numerical columns found.</p>"
    except Exception as e:
        html += f"<p>Error generating numerical stats: {e}</p>"

    # Categorical
    html += "<h3>(b) Categorical/Object Columns Statistics:</h3>"
    try:
        cat_stats = df.describe(include=['object', 'category'])
        if not cat_stats.empty:
            html += cat_stats.to_html(classes='table table-striped', border=1)
        else:
            html += "<p>No categorical/object columns found.</p>"
    except Exception as e:
        html += f"<p>Error generating categorical stats: {e}</p>"
    return html

def identify_column_types_html(df):
    """Generates HTML listing identified column types."""
    html = "<h2>3. Identifying Column Types</h2>"
    numerical_cols = df.select_dtypes(include=np.number).columns.tolist()
    categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
    datetime_cols = df.select_dtypes(include=['datetime', 'datetime64']).columns.tolist()
    boolean_cols = df.select_dtypes(include=['bool']).columns.tolist()
    other_cols = df.columns.difference(numerical_cols + categorical_cols + datetime_cols + boolean_cols).tolist()

    html += f"<p><b>Numerical Columns ({len(numerical_cols)}):</b> <code>{numerical_cols}</code></p>"
    html += f"<p><b>Categorical Columns ({len(categorical_cols)}):</b> <code>{categorical_cols}</code></p>"
    html += f"<p><b>DateTime Columns ({len(datetime_cols)}):</b> <code>{datetime_cols}</code></p>"
    html += f"<p><b>Boolean Columns ({len(boolean_cols)}):</b> <code>{boolean_cols}</code></p>"
    if other_cols:
        html += f"<p><b>Other/Unclassified Columns ({len(other_cols)}):</b> <code>{other_cols}</code></p>"

    # Store for later use (return them)
    return html, numerical_cols, categorical_cols # Return lists as well

def analyze_missing_values_html(df):
    """Generates HTML for missing value analysis."""
    html = "<h2>4. Missing Value Analysis</h2>"
    missing_values = df.isnull().sum()
    missing_percent = (missing_values / len(df)) * 100
    missing_table = pd.concat([missing_values, missing_percent], axis=1, keys=['Missing Count', 'Missing (%)'])
    missing_table = missing_table[missing_table['Missing Count'] > 0].sort_values('Missing (%)', ascending=False)

    if not missing_table.empty:
        html += "<h3>(a) Columns with Missing Values:</h3>"
        html += missing_table.to_html(classes='table table-striped', border=1, float_format='%.2f')

        # Heatmap
        html += "<h3>(b) Missing Values Heatmap:</h3>"
        try:
            fig, ax = plt.subplots(figsize=(15, 7))
            sns.heatmap(df.isnull(), cbar=False, cmap='viridis', ax=ax)
            ax.set_title('Heatmap of Missing Values per Column')
            img_str = fig_to_base64(fig)
            html += f'<img src="{img_str}" alt="Missing Values Heatmap"><br>'
            html += "<p><i>Consider strategies like imputation or deletion based on the results.</i></p>"
        except Exception as e:
             html += f"<p>Could not generate missing value heatmap. Error: {e}</p>"
    else:
        html += "<p>No missing values found in the dataset. Great!</p>"
    return html

def analyze_univariate_numerical_html(df, numerical_cols):
    """Generates HTML for univariate analysis of numerical columns."""
    html = "<h2>5. Univariate Analysis (Numerical Columns)</h2>"
    html += "<p><i>Analyzing distributions of individual numerical features using Histograms and Box Plots.</i></p>"
    if not numerical_cols:
        html += "<p>No numerical columns found to analyze.</p>"
        return html

    for col in numerical_cols:
        html += f"<h3>Analyzing: '{col}'</h3>"
        try:
            # Create subplots
            fig, axes = plt.subplots(1, 2, figsize=(16, 5)) # 1 row, 2 columns

            # Plot Histogram
            sns.histplot(df[col], kde=True, bins=30, ax=axes[0])
            axes[0].set_title(f'Histogram of {col}')
            axes[0].set_xlabel(col)
            axes[0].set_ylabel('Frequency')

            # Plot Box Plot
            sns.boxplot(y=df[col], ax=axes[1])
            axes[1].set_title(f'Box Plot of {col}')
            axes[1].set_ylabel(col)

            plt.tight_layout()
            img_str = fig_to_base64(fig)
            html += f'<img src="{img_str}" alt="Plots for {col}"><br>'

            # Skewness
            skewness = df[col].skew()
            html += f"<p><b>Skewness:</b> {skewness:.2f} "
            if skewness > 0.5: html += "(Moderately Right-Skewed)"
            elif skewness < -0.5: html += "(Moderately Left-Skewed)"
            else: html += "(Approximately Symmetric)"
            html += "</p><hr>"

        except Exception as e:
            html += f"<p>Could not generate plots for {col}. Error: {e}</p><hr>"

    return html

def analyze_univariate_categorical_html(df, categorical_cols):
    """Generates HTML for univariate analysis of categorical columns."""
    html = "<h2>6. Univariate Analysis (Categorical Columns)</h2>"
    html += "<p><i>Analyzing frequency distributions of individual categorical features using Count Plots.</i></p>"
    if not categorical_cols:
        html += "<p>No categorical/object columns found to analyze.</p>"
        return html

    plot_threshold = 50 # Max unique values for plotting

    for col in categorical_cols:
        html += f"<h3>Analyzing: '{col}'</h3>"
        try:
            unique_count = df[col].nunique()
            html += f"<p><b>Number of Unique Values:</b> {unique_count}</p>"

            if unique_count == 0:
                html += "<p><i>Column has no values.</i></p><hr>"
                continue
            elif unique_count > plot_threshold:
                html += f"<p><i>Skipping plot as unique value count ({unique_count}) exceeds threshold ({plot_threshold}). Showing Top 15 value counts instead.</i></p>"
                top_15_counts = df[col].value_counts().head(15)
                html += "<pre>" + top_15_counts.to_string() + "</pre><hr>"
            else:
                # Plot Count Plot
                fig, ax = plt.subplots(figsize=(10, max(5, unique_count * 0.3))) # Adjust height
                plot_order = df[col].value_counts().index
                sns.countplot(y=df[col], order=plot_order, palette='viridis', ax=ax)
                ax.set_title(f'Frequency Count of {col}')
                ax.set_xlabel('Count')
                ax.set_ylabel(col)
                plt.tight_layout()
                img_str = fig_to_base64(fig)
                html += f'<img src="{img_str}" alt="Count Plot for {col}"><hr>'

        except Exception as e:
            html += f"<p>Could not generate plot/counts for {col}. Error: {e}</p><hr>"

    return html

def analyze_bivariate_numerical_html(df, numerical_cols):
    """Generates HTML for bivariate analysis of numerical columns."""
    html = "<h2>7. Bivariate Analysis (Numerical vs. Numerical)</h2>"
    html += "<p><i>Analyzing relationships between pairs of numerical features using Correlation Matrix and Pair Plots.</i></p>"

    if len(numerical_cols) < 2:
        html += "<p>Need at least two numerical columns for this analysis.</p>"
        return html

    # Correlation Heatmap
    html += "<h3>(a) Correlation Matrix Heatmap:</h3>"
    try:
        correlation_matrix = df[numerical_cols].corr()
        fig, ax = plt.subplots(figsize=(12, 10))
        sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=".2f", linewidths=.5, ax=ax)
        ax.set_title('Correlation Matrix of Numerical Features')
        img_str = fig_to_base64(fig)
        html += f'<img src="{img_str}" alt="Correlation Matrix"><br>'
        html += "<p><i>Interpretation: Values close to +1 indicate strong positive linear correlation, close to -1 indicate strong negative linear correlation, close to 0 indicate weak or no linear correlation.</i></p>"
    except Exception as e:
        html += f"<p>Could not generate correlation heatmap. Error: {e}</p>"

    # Pair Plot
    pairplot_threshold = 7 # Limit features for pairplot
    html += f"<h3>(b) Pair Plot (Threshold: {pairplot_threshold} features):</h3>"
    if len(numerical_cols) <= pairplot_threshold:
        html += f"<p><i>Generating Pair Plot for {len(numerical_cols)} numerical features... (May take a moment)</i></p>"
        try:
            pair_plot_fig = sns.pairplot(df[numerical_cols], diag_kind='kde')
            pair_plot_fig.fig.suptitle('Pair Plot of Numerical Features', y=1.02) # Adjust title position
            # Convert the PairGrid object's figure to base64
            img_str = fig_to_base64(pair_plot_fig.fig)
            html += f'<img src="{img_str}" alt="Pair Plot"><br>'
        except Exception as e:
            html += f"<p>Could not generate pair plot. Error: {e}</p>"
            html += "<p><i>Pairplots can sometimes fail with certain data types or distributions, or if memory is limited.</i></p>"
    else:
        html += f"<p><i>Skipping Pair Plot because the number of numerical features ({len(numerical_cols)}) exceeds the threshold ({pairplot_threshold}).</i></p>"

    return html

def analyze_bivariate_num_cat_html(df, numerical_cols, categorical_cols):
    """Generates HTML for bivariate analysis of numerical vs. categorical columns."""
    html = "<h2>8. Bivariate Analysis (Numerical vs. Categorical)</h2>"
    html += "<p><i>Analyzing distributions of numerical features across different categories using Box Plots.</i></p>"

    if not numerical_cols or not categorical_cols:
        html += "<p>Need both numerical and categorical columns for this analysis.</p>"
        return html

    cat_nunique_threshold = 20
    cats_to_analyze = [col for col in categorical_cols if df[col].nunique() <= cat_nunique_threshold]

    if not cats_to_analyze:
        html += f"<p>No categorical columns with a reasonable number of unique values (<= {cat_nunique_threshold}) found for plotting against numerical features.</p>"
        return html

    html += f"<p><i>Analyzing numerical columns against these categorical columns (max {cat_nunique_threshold} unique values): <code>{cats_to_analyze}</code></i></p>"

    for num_col in numerical_cols:
        for cat_col in cats_to_analyze:
            html += f"<h3>Analyzing: '{num_col}' vs '{cat_col}'</h3>"
            try:
                 # Check if category column has data
                if df[cat_col].isnull().all() or df[cat_col].nunique() == 0:
                    html += f"<p><i>Skipping plot: Categorical column '{cat_col}' has no valid data or only one unique value after dropping NaNs.</i></p><hr>"
                    continue

                fig, ax = plt.subplots(figsize=(12, 6))
                sns.boxplot(x=df[cat_col], y=df[num_col], palette='viridis', ax=ax, order=sorted(df[cat_col].dropna().unique())) # Added order and dropna
                ax.set_title(f'Box Plot of {num_col} by {cat_col}')
                ax.set_xlabel(cat_col)
                ax.set_ylabel(num_col)

                # Rotate x-axis labels if they are long or numerous
                if df[cat_col].nunique() > 5:
                    plt.xticks(rotation=45, ha='right')

                plt.tight_layout()
                img_str = fig_to_base64(fig)
                html += f'<img src="{img_str}" alt="Box plot of {num_col} by {cat_col}"><hr>'

            except Exception as e:
                html += f"<p>Could not generate box plot for '{num_col}' vs '{cat_col}'. Error: {e}</p><hr>"

    return html

def get_analysis_summary_html(df, missing_table_html):
    """Generates HTML for the summary section."""
    html = "<h2>9. Analysis Summary & Next Steps</h2>"
    html += "<p>This automated analysis provided a first look at the dataset's structure, content, distributions, and basic relationships.</p>"
    html += "<h3>Key Observations (Auto-Generated Summary):</h3>"
    html += f"<ul><li>The dataset has <b>{df.shape[0]}</b> rows and <b>{df.shape[1]}</b> columns.</li>"
    # Add more sophisticated summary points based on analysis if desired
    if "Columns with Missing Values" in missing_table_html:
         html += "<li>Missing values were detected (see Section 4 for details).</li>"
    else:
         html += "<li>No missing values were found.</li>"
    html += "<li>Review the plots in Sections 5-8 for insights into distributions and relationships.</li>"
    html += "<li><i>(Note: This is a basic summary. Customize with specific findings based on the generated report.)</i></li></ul>"

    html += "<h3>Potential Next Steps:</h3>"
    html += "<ol>"
    html += "<li><b>Data Cleaning:</b> Address missing values (imputation/deletion), correct data types if needed, handle outliers (if appropriate).</li>"
    html += "<li><b>Feature Engineering:</b> Create new features from existing ones (e.g., extracting date parts, combining categories).</li>"
    html += "<li><b>Deeper Analysis:</b> Explore relationships further (statistical tests, different plots, multivariate analysis).</li>"
    html += "<li><b>Domain-Specific Analysis:</b> Apply subject matter expertise for targeted questions.</li>"
    html += "<li><b>Modeling:</b> Prepare data and build machine learning models if applicable.</li>"
    html += "</ol>"
    return html

def get_bonus_guide_html():
    """Generates HTML for the bonus guide."""
    html = """
    <h2>Bonus: How to Understand & Read Any Dataset</h2>
    <p>Approaching a new dataset systematically:</p>
    <ol>
        <li><strong>Understand the Context:</strong> Source, purpose, data dictionary, timeframe.</li>
        <li><strong>Load and Get a First Look:</strong> Use tools like pandas, check dimensions (`.shape`), peek at data (`.head()`, `.tail()`).</li>
        <li><strong>Examine Metadata and Structure:</strong> Check column names (`.columns`), data types (`.info()`), memory usage. Correct types if necessary.</li>
        <li><strong>Summarize the Data:</strong> Use `.describe()` for numerical (mean, median, std, min/max, quartiles) and categorical (unique count, top value, frequency) summaries. Check `.value_counts()` for specific categories.</li>
        <li><strong>Handle Missing Data:</strong> Identify (`.isnull().sum()`) and quantify missing values. Decide on a strategy (deletion, imputation).</li>
        <li><strong>Visualize (EDA):</strong>
            <ul>
                <li><em>Univariate:</em> Histograms, density plots, box plots (numerical); Count plots (categorical).</li>
                <li><em>Bivariate:</em> Scatter plots, correlation matrix/heatmap (numerical vs. numerical); Box plots, violin plots (numerical vs. categorical); Crosstabs, stacked bars (categorical vs. categorical).</li>
                <li><em>Multivariate:</em> Pair plots, faceting.</li>
            </ul>
        </li>
        <li><strong>Ask Questions:</strong> Formulate specific questions based on context and initial findings.</li>
        <li><strong>Iterate and Document:</strong> Data understanding is iterative. Document findings and decisions.</li>
    </ol>
    """
    return html


# --- Main Gradio Function ---

def generate_eda_report(uploaded_file):
    """
    Main function called by Gradio. Takes an uploaded file, performs EDA,
    and returns the path to a generated HTML report file.
    """
    start_time = datetime.now()
    if uploaded_file is None:
        raise gr.Error("No file uploaded! Please upload a CSV file.")

    try:
        # Set visualization styles globally for the run
        sns.set(style="whitegrid")
        plt.rcParams['figure.figsize'] = (12, 6)
        pd.set_option('display.max_columns', 50)
        pd.set_option('display.float_format', lambda x: '%.2f' % x)

        # Check file size (example: 100MB limit)
        file_size_mb = os.path.getsize(uploaded_file.name) / (1024 * 1024)
        if file_size_mb > 100:
            raise gr.Error(f"File size ({file_size_mb:.2f} MB) exceeds the 100 MB limit.")

        # Read the CSV file
        # Use the temporary path provided by Gradio's File component
        df = pd.read_csv(uploaded_file.name)

        # Start building the HTML report
        html_content = """
        <!DOCTYPE html>
        <html lang="en">
        <head>
            <meta charset="UTF-8">
            <meta name="viewport" content="width=device-width, initial-scale=1.0">
            <title>Automated EDA Report</title>
            <style>
                body { font-family: sans-serif; margin: 20px; }
                h1, h2, h3 { color: #333; }
                h1 { text-align: center; border-bottom: 2px solid #eee; padding-bottom: 10px; }
                h2 { border-bottom: 1px solid #eee; padding-bottom: 5px; margin-top: 30px; }
                h3 { margin-top: 20px; color: #555; }
                table { border-collapse: collapse; width: auto; margin-top: 15px; margin-bottom: 15px; }
                th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
                th { background-color: #f2f2f2; }
                tr:nth-child(even) { background-color: #f9f9f9; }
                pre { background-color: #f5f5f5; padding: 10px; border: 1px solid #ccc; overflow-x: auto; }
                code { background-color: #eee; padding: 2px 4px; border-radius: 3px; }
                img { max-width: 100%; height: auto; display: block; margin: 15px auto; border: 1px solid #ddd; }
                hr { border: 0; height: 1px; background: #ddd; margin: 30px 0; }
            </style>
        </head>
        <body>
            <h1>📊 Automated Data Explorer & Visualizer Report 📊</h1>
        """
        report_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        html_content += f"<p style='text-align:center;'><i>Report generated on: {report_time}</i></p>"
        html_content += f"<p style='text-align:center;'><i>Input file: {os.path.basename(uploaded_file.name)}</i></p>"


        # --- Run EDA Steps ---
        # 1. Initial Inspection
        html_content += get_initial_inspection_html(df)
        html_content += "<hr>"

        # 2. Descriptive Statistics
        html_content += get_descriptive_stats_html(df)
        html_content += "<hr>"

        # 3. Identify Column Types
        col_types_html, num_cols, cat_cols = identify_column_types_html(df)
        html_content += col_types_html
        html_content += "<hr>"

        # 4. Missing Values
        missing_html = analyze_missing_values_html(df)
        html_content += missing_html
        html_content += "<hr>"

        # 5. Univariate Numerical
        html_content += analyze_univariate_numerical_html(df, num_cols)
        html_content += "<hr>"

        # 6. Univariate Categorical
        html_content += analyze_univariate_categorical_html(df, cat_cols)
        html_content += "<hr>"

        # 7. Bivariate Numerical vs Numerical
        html_content += analyze_bivariate_numerical_html(df, num_cols)
        html_content += "<hr>"

        # 8. Bivariate Numerical vs Categorical
        html_content += analyze_bivariate_num_cat_html(df, num_cols, cat_cols)
        html_content += "<hr>"

        # 9. Summary
        html_content += get_analysis_summary_html(df, missing_html) # Pass missing_html to check if missing values were found
        html_content += "<hr>"

        # 10. Bonus Guide
        html_content += get_bonus_guide_html()

        # --- Finalize HTML ---
        html_content += f"<p style='text-align:center; margin-top: 30px;'><i>--- End of Report ---</i></p>"
        end_time = datetime.now()
        duration = end_time - start_time
        html_content += f"<p style='text-align:center; font-size: small; color: grey;'><i>Analysis completed in {duration.total_seconds():.2f} seconds.</i></p>"
        html_content += """
            </body>
            </html>
            """

        # Save HTML content to a temporary file
        # Use tempfile for better cross-platform compatibility and automatic cleanup
        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=".html", encoding='utf-8') as temp_file:
            temp_file.write(html_content)
            report_path = temp_file.name # Get the path of the temp file

        # Return the path to the generated HTML file for Gradio output
        return report_path

    except pd.errors.ParserError:
        raise gr.Error("Error parsing CSV file. Please ensure it is a valid CSV format and delimiter is correctly inferred (usually comma).")
    except FileNotFoundError:
         raise gr.Error("Uploaded file not found. Please try uploading again.")
    except ValueError as ve: # Catch specific value errors like Colab's upload error
         raise gr.Error(f"Value Error: {ve}")
    except Exception as e:
        # Generic error catch - useful for debugging
        import traceback
        tb_str = traceback.format_exc()
        print(f"An unexpected error occurred: {e}\n{tb_str}") # Log to console
        raise gr.Error(f"An unexpected error occurred during analysis: {e}. Check console logs if running locally.")


# --- Gradio Interface Setup ---

description = """
**Effortless Dataset Insights 📊**

Upload your CSV dataset (max 100MB) and get an automated Exploratory Data Analysis (EDA) report.
The report includes:
1. Basic Info (Shape, Data Types, Head/Tail)
2. Descriptive Statistics
3. Missing Value Analysis & Heatmap
4. Univariate Analysis (Histograms, Box Plots, Count Plots)
5. Bivariate Analysis (Correlation Heatmap, Pair Plot [small datasets], Box Plots by Category)
6. Summary & Next Steps Guide

The output will be an HTML file that you can download and view in your browser.
"""

iface = gr.Interface(
    fn=generate_eda_report,
    inputs=gr.File(label="Upload CSV Dataset", file_types=[".csv"]),
    outputs=gr.File(label="Download EDA Report (.html)"),
    title="Effortless Dataset Insights",
    description=description,
    allow_flagging="never",
    examples=[
        # You can add paths to example CSV files here if you host them somewhere
        # e.g., ["./examples/sample_data.csv"]
        # Ensure these files exist if you uncomment this
    ],
     theme=gr.themes.Soft() # Optional: Apply a theme
)

# --- Launch the App ---
if __name__ == "__main__":
    iface.launch()