Update app.py

app.py

@@ -2,48 +2,41 @@
 
 import streamlit as st
 import numpy as np
-import pandas as pd
+import pandas as pd 
 from smolagents import CodeAgent, tool
 from typing import Union, List, Dict, Optional
 import matplotlib.pyplot as plt
 import seaborn as sns
-import plotly.express as px
-import plotly.graph_objects as go
 import os
 from groq import Groq
 from dataclasses import dataclass
 import tempfile
 import base64
-import io
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc
-import joblib
-import pdfkit  # Ensure wkhtmltopdf is available in the environment
-import uuid  # For generating unique report IDs
+import io 
 
 # ------------------------------
 # Language Model Interface
 # ------------------------------
 class GroqLLM:
-    """Enhanced LLM interface with support for generating natural language summaries."""
-
+    """Compatible LLM interface for smolagents CodeAgent"""
+    
     def __init__(self, model_name: str = "llama-3.1-8B-Instant"):
         """
-        Initialize the GroqLLM with a specified model.
-
+        Initialize the GroqLLM with the specified model.
+        
         Args:
             model_name (str): The name of the language model to use.
         """
         self.client = Groq(api_key=os.environ.get("GROQ_API_KEY"))
         self.model_name = model_name
-
+    
     def __call__(self, prompt: Union[str, dict, List[Dict]]) -> str:
         """
         Make the class callable as required by smolagents.
-
+        
         Args:
             prompt (Union[str, dict, List[Dict]]): The input prompt for the language model.
-
+        
         Returns:
             str: The generated response from the language model.
         """
@@ -62,7 +55,7 @@ class GroqLLM:
                     "content": prompt_str
                 }],
                 temperature=0.7,
-                max_tokens=1500,  # Increased tokens for detailed responses
+                max_tokens=1024,
                 stream=False
             )
             
@@ -77,12 +70,12 @@ class GroqLLM:
 # Data Analysis Agent
 # ------------------------------
 class DataAnalysisAgent(CodeAgent):
-    """Extended CodeAgent with dataset awareness and predictive analytics capabilities."""
-
+    """Extended CodeAgent with dataset awareness"""
+    
     def __init__(self, dataset: pd.DataFrame, *args, **kwargs):
         """
         Initialize the DataAnalysisAgent with the provided dataset.
-
+        
         Args:
             dataset (pd.DataFrame): The dataset to analyze.
             *args: Variable length argument list.
@@ -90,24 +83,19 @@ class DataAnalysisAgent(CodeAgent):
         """
         super().__init__(*args, **kwargs)
         self._dataset = dataset
-        self.models = {}  # To store trained models
-
+    
     @property
     def dataset(self) -> pd.DataFrame:
-        """Access the stored dataset.
-
-        Returns:
-            pd.DataFrame: The dataset stored in the agent.
-        """
+        """Access the stored dataset."""
         return self._dataset
-
+    
     def run(self, prompt: str) -> str:
         """
-        Override the run method to include dataset context and support predictive tasks.
-
+        Override run method to include dataset context.
+        
         Args:
             prompt (str): The task prompt for analysis.
-
+        
         Returns:
             str: The result of the analysis.
         """
@@ -133,23 +121,22 @@ class DataAnalysisAgent(CodeAgent):
 @tool
 def analyze_basic_stats(data: Optional[pd.DataFrame] = None) -> str:
     """
-    Calculate and visualize basic statistical measures for numerical columns.
-
+    Calculate basic statistical measures for numerical columns in the dataset.
+    
     This function computes fundamental statistical metrics including mean, median,
     standard deviation, skewness, and counts of missing values for all numerical
-    columns in the provided DataFrame. It also generates a bar chart visualizing
-    the mean, median, and standard deviation for each numerical feature.
-
+    columns in the provided DataFrame.
+    
     Args:
         data (Optional[pd.DataFrame], optional):
-            A pandas DataFrame containing the dataset to analyze. 
-            If None, the agent's stored dataset will be used.
-            The DataFrame should contain at least one numerical column
-            for meaningful analysis.
-
+            A pandas DataFrame containing the dataset to analyze. The DataFrame
+            should contain at least one numerical column for meaningful analysis.
+    
     Returns:
-        str: A markdown-formatted string containing the statistics and the generated plot.
+        str: A string containing formatted basic statistics for each numerical column,
+             including mean, median, standard deviation, skewness, and missing value counts.
     """
+    # Access dataset from agent if no data provided
     if data is None:
         data = tool.agent.dataset
     
@@ -165,280 +152,110 @@ def analyze_basic_stats(data: Optional[pd.DataFrame] = None) -> str:
             'missing': int(data[col].isnull().sum())
         }
     
-    # Generate a summary DataFrame
-    stats_df = pd.DataFrame(stats).T
-    stats_df.reset_index(inplace=True)
-    stats_df.rename(columns={'index': 'Feature'}, inplace=True)
-    
-    # Plotting basic statistics
-    fig, ax = plt.subplots(figsize=(10, 6))
-    stats_df.set_index('Feature')[['mean', 'median', 'std']].plot(kind='bar', ax=ax)
-    plt.title('Basic Statistics')
-    plt.ylabel('Values')
-    plt.tight_layout()
-    
-    # Save plot to buffer
-    buf = io.BytesIO()
-    plt.savefig(buf, format='png')
-    plt.close()
-    stats_plot = base64.b64encode(buf.getvalue()).decode()
-    
-    return f"### Basic Statistics\n{stats_df.to_markdown()} \n\n![Basic Statistics](data:image/png;base64,{stats_plot})"
+    return str(stats)
 
 @tool
 def generate_correlation_matrix(data: Optional[pd.DataFrame] = None) -> str:
     """
-    Generate an interactive correlation matrix using Plotly.
-
-    This function creates an interactive heatmap visualization showing the correlations between
-    all numerical columns in the dataset. Users can hover over cells to see correlation values
-    and interact with the plot (zoom, pan).
-
+    Generate a visual correlation matrix for numerical columns in the dataset.
+    
+    This function creates a heatmap visualization showing the correlations between
+    all numerical columns in the dataset. The correlation values are displayed
+    using a color-coded matrix for easy interpretation.
+    
     Args:
         data (Optional[pd.DataFrame], optional):
-            A pandas DataFrame containing the dataset to analyze. 
-            If None, the agent's stored dataset will be used.
-            The DataFrame should contain at least two numerical columns
-            for correlation analysis.
-
+            A pandas DataFrame containing the dataset to analyze. The DataFrame
+            should contain at least two numerical columns for correlation analysis.
+    
     Returns:
-        str: An HTML string representing the interactive correlation matrix plot.
+        str: A base64 encoded string representing the correlation matrix plot image,
+             which can be displayed in a web interface or saved as an image file.
     """
+    # Access dataset from agent if no data provided
     if data is None:
         data = tool.agent.dataset
-    
+        
     numeric_data = data.select_dtypes(include=[np.number])
-    corr = numeric_data.corr()
-    
-    fig = px.imshow(corr,
-                    text_auto=True,
-                    aspect="auto",
-                    color_continuous_scale='RdBu',
-                    title='Correlation Matrix')
     
-    fig.update_layout(width=800, height=600)
+    plt.figure(figsize=(10, 8))
+    sns.heatmap(numeric_data.corr(), annot=True, cmap='coolwarm')
+    plt.title('Correlation Matrix')
     
-    # Convert Plotly figure to HTML div
-    correlation_html = fig.to_html(full_html=False)
-    
-    return correlation_html
+    buf = io.BytesIO()
+    plt.savefig(buf, format='png')
+    plt.close()
+    return base64.b64encode(buf.getvalue()).decode()
 
 @tool
 def analyze_categorical_columns(data: Optional[pd.DataFrame] = None) -> str:
     """
-    Analyze categorical columns with visualizations.
-
+    Analyze categorical columns in the dataset for distribution and frequencies.
+    
     This function examines categorical columns to identify unique values, top categories,
-    and missing value counts. It also generates bar charts for the top 5 categories in each
-    categorical feature.
-
+    and missing value counts, providing insights into the categorical data distribution.
+    
     Args:
         data (Optional[pd.DataFrame], optional):
-            A pandas DataFrame containing the dataset to analyze. 
-            If None, the agent's stored dataset will be used.
-            The DataFrame should contain at least one categorical column
-            for meaningful analysis.
-
+            A pandas DataFrame containing the dataset to analyze. The DataFrame
+            should contain at least one categorical column for meaningful analysis.
+    
     Returns:
-        str: A markdown-formatted string containing analysis results and embedded plots.
+        str: A string containing formatted analysis results for each categorical column,
+             including unique value counts, top categories, and missing value counts.
     """
+    # Access dataset from agent if no data provided
     if data is None:
         data = tool.agent.dataset
-    
+        
     categorical_cols = data.select_dtypes(include=['object', 'category']).columns
     analysis = {}
-    plots = ""
     
     for col in categorical_cols:
-        unique_vals = data[col].nunique()
-        top_categories = data[col].value_counts().head(5).to_dict()
-        missing = data[col].isnull().sum()
-        
         analysis[col] = {
-            'unique_values': int(unique_vals),
-            'top_categories': top_categories,
-            'missing': int(missing)
+            'unique_values': int(data[col].nunique()),
+            'top_categories': data[col].value_counts().head(5).to_dict(),
+            'missing': int(data[col].isnull().sum())
         }
-        
-        # Generate bar chart for top categories
-        fig, ax = plt.subplots(figsize=(8, 4))
-        sns.countplot(data=data, x=col, order=data[col].value_counts().iloc[:5].index, ax=ax)
-        plt.title(f'Top 5 Categories in {col}')
-        plt.xticks(rotation=45)
-        plt.tight_layout()
-        
-        buf = io.BytesIO()
-        plt.savefig(buf, format='png')
-        plt.close()
-        plot_img = base64.b64encode(buf.getvalue()).decode()
-        
-        plots += f"### {col}\n"
-        plots += f"- **Unique Values:** {unique_vals}\n"
-        plots += f"- **Missing Values:** {missing}\n"
-        plots += f"- **Top Categories:** {top_categories}\n"
-        plots += f"![Top Categories in {col}](data:image/png;base64,{plot_img})\n\n"
     
-    return plots + f"### Categorical Columns Analysis\n{pd.DataFrame(analysis).T.to_markdown()}"
+    return str(analysis)
 
 @tool
 def suggest_features(data: Optional[pd.DataFrame] = None) -> str:
     """
     Suggest potential feature engineering steps based on data characteristics.
-
+    
     This function analyzes the dataset's structure and statistical properties to
     recommend possible feature engineering steps that could improve model performance.
-
+    
     Args:
         data (Optional[pd.DataFrame], optional):
-            A pandas DataFrame containing the dataset to analyze. 
-            If None, the agent's stored dataset will be used.
-            The DataFrame can contain both numerical and categorical columns.
-
+            A pandas DataFrame containing the dataset to analyze. The DataFrame
+            can contain both numerical and categorical columns.
+    
     Returns:
         str: A string containing suggestions for feature engineering based on
              the characteristics of the input data.
     """
+    # Access dataset from agent if no data provided
     if data is None:
         data = tool.agent.dataset
-    
+        
     suggestions = []
     numeric_cols = data.select_dtypes(include=[np.number]).columns
     categorical_cols = data.select_dtypes(include=['object', 'category']).columns
     
-    # Interaction terms
     if len(numeric_cols) >= 2:
-        suggestions.append("• **Interaction Terms:** Consider creating interaction terms between numerical features to capture combined effects.")
+        suggestions.append("Consider creating interaction terms between numerical features")
     
-    # Encoding categorical variables
     if len(categorical_cols) > 0:
-        suggestions.append("• **One-Hot Encoding:** Apply one-hot encoding to categorical variables to convert them into numerical format.")
-        suggestions.append("• **Label Encoding:** For ordinal categorical variables, consider label encoding to maintain order information.")
-    
-    # Handling skewness
+        suggestions.append("Consider one-hot encoding for categorical variables")
+        
     for col in numeric_cols:
         if data[col].skew() > 1 or data[col].skew() < -1:
-            suggestions.append(f"• **Log Transformation:** Apply log transformation to `{col}` to reduce skewness and stabilize variance.")
-    
-    # Missing value imputation
-    for col in data.columns:
-        if data[col].isnull().sum() > 0:
-            suggestions.append(f"• **Imputation:** Consider imputing missing values in `{col}` using mean, median, or advanced imputation techniques.")
-    
-    # Feature scaling
-    suggestions.append("• **Feature Scaling:** Apply feature scaling (Standardization or Normalization) to numerical features to ensure uniformity.")
-    
-    return "\n".join(suggestions)
-
-@tool
-def predictive_analysis(data: Optional[pd.DataFrame] = None, target: Optional[str] = None) -> str:
-    """
-    Perform predictive analytics by training a classification model.
-
-    This function builds a classification model using Random Forest, evaluates its performance,
-    and provides detailed metrics and visualizations such as the confusion matrix and ROC curve.
-
-    Args:
-        data (Optional[pd.DataFrame], optional):
-            A pandas DataFrame containing the dataset to analyze. 
-            If None, the agent's stored dataset will be used.
-            The DataFrame should contain the target variable for prediction.
-        target (Optional[str], optional):
-            The name of the target variable column in the dataset.
-            If None, the agent must provide the target variable through the prompt.
-
-    Returns:
-        str: A markdown-formatted string containing the classification report, confusion matrix,
-             ROC curve, AUC score, and a unique Model ID.
-    """
-    if data is None:
-        data = tool.agent.dataset
-    
-    if target is None or target not in data.columns:
-        return f"Error: Target column not specified or `{target}` not found in the dataset."
-    
-    # Handle categorical target
-    if data[target].dtype == 'object' or data[target].dtype.name == 'category':
-        data[target] = data[target].astype('category').cat.codes
-    
-    # Drop rows with missing target
-    data = data.dropna(subset=[target])
-    
-    # Separate features and target
-    X = data.drop(columns=[target])
-    y = data[target]
-    
-    # Handle missing values (simple imputation)
-    X = X.fillna(X.median())
-    
-    # Encode categorical variables
-    X = pd.get_dummies(X, drop_first=True)
+            suggestions.append(f"Consider log transformation for {col} due to skewness")
     
-    # Split data
-    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
-    
-    # Train a Random Forest Classifier (as an example)
-    from sklearn.ensemble import RandomForestClassifier
-    clf = RandomForestClassifier(n_estimators=100, random_state=42)
-    clf.fit(X_train, y_train)
-    
-    # Predictions
-    y_pred = clf.predict(X_test)
-    y_proba = clf.predict_proba(X_test)[:,1]
-    
-    # Evaluation
-    report = classification_report(y_test, y_pred, output_dict=True)
-    report_df = pd.DataFrame(report).transpose()
-    
-    # Confusion Matrix
-    cm = confusion_matrix(y_test, y_pred)
-    fig_cm = px.imshow(cm, text_auto=True, labels=dict(x="Predicted", y="Actual", color="Count"),
-                       x=["Negative", "Positive"], y=["Negative", "Positive"],
-                       title="Confusion Matrix")
-    
-    # ROC Curve
-    fpr, tpr, thresholds = roc_curve(y_test, y_proba)
-    roc_auc = auc(fpr, tpr)
-    fig_roc = go.Figure()
-    fig_roc.add_trace(go.Scatter(x=fpr, y=tpr, mode='lines', name=f'ROC Curve (AUC = {roc_auc:.2f})'))
-    fig_roc.add_trace(go.Scatter(x=[0,1], y=[0,1], mode='lines', name='Random Guess', line=dict(dash='dash')))
-    fig_roc.update_layout(title='Receiver Operating Characteristic (ROC) Curve',
-                          xaxis_title='False Positive Rate',
-                          yaxis_title='True Positive Rate')
-    
-    # Save models for potential future use
-    model_id = str(uuid.uuid4())
-    with tempfile.NamedTemporaryFile(delete=False, suffix='.joblib') as tmp_model_file:
-        joblib.dump(clf, tmp_model_file.name)
-        # In a real-world scenario, you'd store this in a persistent storage
-    tool.agent.models[model_id] = clf  # Storing in agent's models dict
-    
-    # Generate HTML for plots
-    cm_html = fig_cm.to_html(full_html=False)
-    roc_html = fig_roc.to_html(full_html=False)
-    
-    # Generate report summary
-    summary = f"""
-    ### Predictive Analytics Report for Target: `{target}`
-    
-    **Model Used:** Random Forest Classifier
-    
-    **Classification Report:**
-    {report_df.to_markdown()}
-    
-    **Confusion Matrix:**
-    {cm_html}
-    
-    **ROC Curve:**
-    {roc_html}
-    
-    **AUC Score:** {roc_auc:.2f}
-    
-    **Model ID:** `{model_id}`
-    
-    *You can use this Model ID to retrieve or update the model in future analyses.*
-    """
-    
-    return summary
+    return '\n'.join(suggestions)
 
 # ------------------------------
 # Report Exporting Function
@@ -446,14 +263,14 @@ def predictive_analysis(data: Optional[pd.DataFrame] = None, target: Optional[st
 def export_report(content: str, filename: str):
     """
     Export the given content as a PDF report.
-
+    
     This function converts markdown content into a PDF file using pdfkit and provides
     a download button for users to obtain the report.
-
+    
     Args:
         content (str): The markdown content to be included in the PDF report.
         filename (str): The desired name for the exported PDF file.
-
+    
     Returns:
         None
     """
@@ -491,7 +308,7 @@ def export_report(content: str, filename: str):
 def main():
     st.set_page_config(page_title="📊 Business Intelligence Assistant", layout="wide")
     st.title("📊 **Business Intelligence Assistant**")
-    st.write("Upload your dataset and receive comprehensive analyses, interactive visualizations, and predictive insights.")
+    st.write("Upload your dataset and get automated analysis with natural language interaction.")
     
     # Initialize session state
     if 'data' not in st.session_state:
@@ -501,8 +318,7 @@ def main():
     if 'report_content' not in st.session_state:
         st.session_state['report_content'] = ""
     
-    # File Uploader
-    uploaded_file = st.file_uploader("📥 **Upload a CSV file**", type="csv")
+    uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
     
     try:
         if uploaded_file is not None:
@@ -515,93 +331,80 @@ def main():
                 st.session_state['agent'] = DataAnalysisAgent(
                     dataset=data,
                     tools=[analyze_basic_stats, generate_correlation_matrix, 
-                           analyze_categorical_columns, suggest_features, predictive_analysis],
+                           analyze_categorical_columns, suggest_features],
                     model=GroqLLM(),
-                    additional_authorized_imports=["pandas", "numpy", "matplotlib", "seaborn", "plotly"]
+                    additional_authorized_imports=["pandas", "numpy", "matplotlib", "seaborn"]
                 )
                 
-                st.success(f"✅ Successfully loaded dataset with {data.shape[0]} rows and {data.shape[1]} columns")
+                st.success(f'Successfully loaded dataset with {data.shape[0]} rows and {data.shape[1]} columns')
                 st.subheader("🔍 **Data Preview**")
                 st.dataframe(data.head())
         
         if st.session_state['data'] is not None:
-            # Sidebar for Analysis Selection
-            st.sidebar.header("🛠️ **Select Analysis Type**")
-            analysis_type = st.sidebar.selectbox(
+            analysis_type = st.selectbox(
                 "Choose analysis type",
                 ["Basic Statistics", "Correlation Analysis", "Categorical Analysis", 
-                 "Feature Engineering", "Predictive Analytics", "Custom Question"]
+                 "Feature Engineering", "Custom Question"]
             )
             
             if analysis_type == "Basic Statistics":
-                with st.spinner('📈 Analyzing basic statistics...'):
+                with st.spinner('Analyzing basic statistics...'):
                     result = st.session_state['agent'].run(
                         "Use the analyze_basic_stats tool to analyze this dataset and "
                         "provide insights about the numerical distributions."
                     )
-                    st.markdown(result, unsafe_allow_html=True)
+                    st.write(result)
                     st.session_state['report_content'] += result + "\n\n"
-            
+                    
             elif analysis_type == "Correlation Analysis":
-                with st.spinner('📊 Generating correlation matrix...'):
+                with st.spinner('Generating correlation matrix...'):
                     result = st.session_state['agent'].run(
                         "Use the generate_correlation_matrix tool to analyze correlations "
                         "and explain any strong relationships found."
                     )
-                    st.components.v1.html(result, height=600)
+                    if isinstance(result, str) and 'base64' in result:
+                        # Extract base64 string and display the image
+                        image_data = f"data:image/png;base64,{result}"
+                        st.image(image_data, caption='Correlation Matrix')
+                    else:
+                        st.write(result)
                     st.session_state['report_content'] += "### Correlation Analysis\n" + result + "\n\n"
-            
+                    
             elif analysis_type == "Categorical Analysis":
-                with st.spinner('📊 Analyzing categorical columns...'):
+                with st.spinner('Analyzing categorical columns...'):
                     result = st.session_state['agent'].run(
                         "Use the analyze_categorical_columns tool to examine the "
                         "categorical variables and explain the distributions."
                     )
-                    st.markdown(result, unsafe_allow_html=True)
-                    st.session_state['report_content'] += result + "\n\n"
-            
+                    st.write(result)
+                    st.session_state['report_content'] += "### Categorical Analysis\n" + result + "\n\n"
+                    
             elif analysis_type == "Feature Engineering":
-                with st.spinner('🔧 Generating feature suggestions...'):
+                with st.spinner('Generating feature suggestions...'):
                     result = st.session_state['agent'].run(
                         "Use the suggest_features tool to recommend potential "
                         "feature engineering steps for this dataset."
                     )
-                    st.markdown(result, unsafe_allow_html=True)
-                    st.session_state['report_content'] += result + "\n\n"
-            
-            elif analysis_type == "Predictive Analytics":
-                with st.form("Predictive Analytics Form"):
-                    st.write("🔮 **Predictive Analytics**")
-                    target = st.selectbox("Select the target variable for prediction:", options=st.session_state['data'].columns)
-                    submit = st.form_submit_button("🚀 Run Predictive Analysis")
-                
-                if submit:
-                    with st.spinner('🚀 Performing predictive analysis...'):
-                        result = st.session_state['agent'].run(
-                            f"Use the predictive_analysis tool to build a classification model with `{target}` as the target variable."
-                        )
-                        st.markdown(result, unsafe_allow_html=True)
-                        st.session_state['report_content'] += result + "\n\n"
-                        export_report(result, "Predictive_Analysis_Report")
-            
+                    st.write(result)
+                    st.session_state['report_content'] += "### Feature Engineering Suggestions\n" + result + "\n\n"
+                    
             elif analysis_type == "Custom Question":
-                with st.expander("📝 **Ask a Custom Question**"):
-                    question = st.text_input("What would you like to know about your data?")
-                    if st.button("🔍 Get Answer"):
-                        if question:
-                            with st.spinner('🧠 Processing your question...'):
-                                result = st.session_state['agent'].run(question)
-                                st.markdown(result, unsafe_allow_html=True)
-                                st.session_state['report_content'] += f"### Custom Question: {question}\n{result}\n\n"
-                        else:
-                            st.warning("Please enter a question.")
+                question = st.text_input("What would you like to know about your data?")
+                if st.button("🔍 Get Answer"):
+                    if question:
+                        with st.spinner('Analyzing...'):
+                            result = st.session_state['agent'].run(question)
+                            st.write(result)
+                            st.session_state['report_content'] += f"### Custom Question: {question}\n{result}\n\n"
+                    else:
+                        st.warning("Please enter a question.")
             
             # Option to Export Report
             if st.session_state['report_content']:
-                st.sidebar.markdown("---")
-                if st.sidebar.button("📤 **Export Analysis Report**"):
+                st.markdown("---")
+                if st.button("📤 **Export Analysis Report**"):
                     export_report(st.session_state['report_content'], "Business_Intelligence_Report")
-                    st.sidebar.success("✅ Report exported successfully!")
+                    st.success("✅ Report exported successfully!")
     
     except Exception as e:
         st.error(f"⚠️ An error occurred: {str(e)}")