Spaces:
Build error
Build error
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,506 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pandas as pd
|
2 |
+
import numpy as np
|
3 |
+
import matplotlib.pyplot as plt
|
4 |
+
import seaborn as sns
|
5 |
+
import gradio as gr
|
6 |
+
import io
|
7 |
+
import base64
|
8 |
+
import tempfile
|
9 |
+
import os
|
10 |
+
from datetime import datetime
|
11 |
+
|
12 |
+
# --- Matplotlib Plot to Base64 ---
|
13 |
+
def fig_to_base64(fig):
|
14 |
+
"""Converts a Matplotlib figure to a base64 encoded PNG string."""
|
15 |
+
buf = io.BytesIO()
|
16 |
+
fig.savefig(buf, format='png', bbox_inches='tight')
|
17 |
+
plt.close(fig) # Close the figure to free memory
|
18 |
+
buf.seek(0)
|
19 |
+
img_str = base64.b64encode(buf.read()).decode('utf-8')
|
20 |
+
return f"data:image/png;base64,{img_str}"
|
21 |
+
|
22 |
+
# --- EDA Helper Functions (Adapted from Colab) ---
|
23 |
+
|
24 |
+
def get_initial_inspection_html(df):
|
25 |
+
"""Generates HTML for initial data inspection."""
|
26 |
+
html = "<h2>1. Initial Data Inspection</h2>"
|
27 |
+
# Head
|
28 |
+
html += "<h3>(a) First 5 Rows (Head):</h3>"
|
29 |
+
html += df.head().to_html(classes='table table-striped', border=1)
|
30 |
+
# Tail
|
31 |
+
html += "<h3>(b) Last 5 Rows (Tail):</h3>"
|
32 |
+
html += df.tail().to_html(classes='table table-striped', border=1)
|
33 |
+
# Shape
|
34 |
+
html += "<h3>(c) Dataset Shape:</h3>"
|
35 |
+
html += f"<p>Number of Rows: {df.shape[0]}</p>"
|
36 |
+
html += f"<p>Number of Columns: {df.shape[1]}</p>"
|
37 |
+
# Info
|
38 |
+
html += "<h3>(d) Data Types and Non-Null Counts (Info):</h3>"
|
39 |
+
buffer = io.StringIO()
|
40 |
+
df.info(buf=buffer)
|
41 |
+
info_str = buffer.getvalue()
|
42 |
+
html += f"<pre>{info_str}</pre>"
|
43 |
+
# Column Names
|
44 |
+
html += "<h3>(e) Column Names:</h3>"
|
45 |
+
html += f"<p><code>{list(df.columns)}</code></p>"
|
46 |
+
return html
|
47 |
+
|
48 |
+
def get_descriptive_stats_html(df):
|
49 |
+
"""Generates HTML for descriptive statistics."""
|
50 |
+
html = "<h2>2. Descriptive Statistics</h2>"
|
51 |
+
# Numerical
|
52 |
+
html += "<h3>(a) Numerical Columns Statistics:</h3>"
|
53 |
+
try:
|
54 |
+
num_stats = df.describe(include=np.number)
|
55 |
+
if not num_stats.empty:
|
56 |
+
html += num_stats.to_html(classes='table table-striped', border=1, float_format='%.2f')
|
57 |
+
else:
|
58 |
+
html += "<p>No numerical columns found.</p>"
|
59 |
+
except Exception as e:
|
60 |
+
html += f"<p>Error generating numerical stats: {e}</p>"
|
61 |
+
|
62 |
+
# Categorical
|
63 |
+
html += "<h3>(b) Categorical/Object Columns Statistics:</h3>"
|
64 |
+
try:
|
65 |
+
cat_stats = df.describe(include=['object', 'category'])
|
66 |
+
if not cat_stats.empty:
|
67 |
+
html += cat_stats.to_html(classes='table table-striped', border=1)
|
68 |
+
else:
|
69 |
+
html += "<p>No categorical/object columns found.</p>"
|
70 |
+
except Exception as e:
|
71 |
+
html += f"<p>Error generating categorical stats: {e}</p>"
|
72 |
+
return html
|
73 |
+
|
74 |
+
def identify_column_types_html(df):
|
75 |
+
"""Generates HTML listing identified column types."""
|
76 |
+
html = "<h2>3. Identifying Column Types</h2>"
|
77 |
+
numerical_cols = df.select_dtypes(include=np.number).columns.tolist()
|
78 |
+
categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
|
79 |
+
datetime_cols = df.select_dtypes(include=['datetime', 'datetime64']).columns.tolist()
|
80 |
+
boolean_cols = df.select_dtypes(include=['bool']).columns.tolist()
|
81 |
+
other_cols = df.columns.difference(numerical_cols + categorical_cols + datetime_cols + boolean_cols).tolist()
|
82 |
+
|
83 |
+
html += f"<p><b>Numerical Columns ({len(numerical_cols)}):</b> <code>{numerical_cols}</code></p>"
|
84 |
+
html += f"<p><b>Categorical Columns ({len(categorical_cols)}):</b> <code>{categorical_cols}</code></p>"
|
85 |
+
html += f"<p><b>DateTime Columns ({len(datetime_cols)}):</b> <code>{datetime_cols}</code></p>"
|
86 |
+
html += f"<p><b>Boolean Columns ({len(boolean_cols)}):</b> <code>{boolean_cols}</code></p>"
|
87 |
+
if other_cols:
|
88 |
+
html += f"<p><b>Other/Unclassified Columns ({len(other_cols)}):</b> <code>{other_cols}</code></p>"
|
89 |
+
|
90 |
+
# Store for later use (return them)
|
91 |
+
return html, numerical_cols, categorical_cols # Return lists as well
|
92 |
+
|
93 |
+
def analyze_missing_values_html(df):
|
94 |
+
"""Generates HTML for missing value analysis."""
|
95 |
+
html = "<h2>4. Missing Value Analysis</h2>"
|
96 |
+
missing_values = df.isnull().sum()
|
97 |
+
missing_percent = (missing_values / len(df)) * 100
|
98 |
+
missing_table = pd.concat([missing_values, missing_percent], axis=1, keys=['Missing Count', 'Missing (%)'])
|
99 |
+
missing_table = missing_table[missing_table['Missing Count'] > 0].sort_values('Missing (%)', ascending=False)
|
100 |
+
|
101 |
+
if not missing_table.empty:
|
102 |
+
html += "<h3>(a) Columns with Missing Values:</h3>"
|
103 |
+
html += missing_table.to_html(classes='table table-striped', border=1, float_format='%.2f')
|
104 |
+
|
105 |
+
# Heatmap
|
106 |
+
html += "<h3>(b) Missing Values Heatmap:</h3>"
|
107 |
+
try:
|
108 |
+
fig, ax = plt.subplots(figsize=(15, 7))
|
109 |
+
sns.heatmap(df.isnull(), cbar=False, cmap='viridis', ax=ax)
|
110 |
+
ax.set_title('Heatmap of Missing Values per Column')
|
111 |
+
img_str = fig_to_base64(fig)
|
112 |
+
html += f'<img src="{img_str}" alt="Missing Values Heatmap"><br>'
|
113 |
+
html += "<p><i>Consider strategies like imputation or deletion based on the results.</i></p>"
|
114 |
+
except Exception as e:
|
115 |
+
html += f"<p>Could not generate missing value heatmap. Error: {e}</p>"
|
116 |
+
else:
|
117 |
+
html += "<p>No missing values found in the dataset. Great!</p>"
|
118 |
+
return html
|
119 |
+
|
120 |
+
def analyze_univariate_numerical_html(df, numerical_cols):
|
121 |
+
"""Generates HTML for univariate analysis of numerical columns."""
|
122 |
+
html = "<h2>5. Univariate Analysis (Numerical Columns)</h2>"
|
123 |
+
html += "<p><i>Analyzing distributions of individual numerical features using Histograms and Box Plots.</i></p>"
|
124 |
+
if not numerical_cols:
|
125 |
+
html += "<p>No numerical columns found to analyze.</p>"
|
126 |
+
return html
|
127 |
+
|
128 |
+
for col in numerical_cols:
|
129 |
+
html += f"<h3>Analyzing: '{col}'</h3>"
|
130 |
+
try:
|
131 |
+
# Create subplots
|
132 |
+
fig, axes = plt.subplots(1, 2, figsize=(16, 5)) # 1 row, 2 columns
|
133 |
+
|
134 |
+
# Plot Histogram
|
135 |
+
sns.histplot(df[col], kde=True, bins=30, ax=axes[0])
|
136 |
+
axes[0].set_title(f'Histogram of {col}')
|
137 |
+
axes[0].set_xlabel(col)
|
138 |
+
axes[0].set_ylabel('Frequency')
|
139 |
+
|
140 |
+
# Plot Box Plot
|
141 |
+
sns.boxplot(y=df[col], ax=axes[1])
|
142 |
+
axes[1].set_title(f'Box Plot of {col}')
|
143 |
+
axes[1].set_ylabel(col)
|
144 |
+
|
145 |
+
plt.tight_layout()
|
146 |
+
img_str = fig_to_base64(fig)
|
147 |
+
html += f'<img src="{img_str}" alt="Plots for {col}"><br>'
|
148 |
+
|
149 |
+
# Skewness
|
150 |
+
skewness = df[col].skew()
|
151 |
+
html += f"<p><b>Skewness:</b> {skewness:.2f} "
|
152 |
+
if skewness > 0.5: html += "(Moderately Right-Skewed)"
|
153 |
+
elif skewness < -0.5: html += "(Moderately Left-Skewed)"
|
154 |
+
else: html += "(Approximately Symmetric)"
|
155 |
+
html += "</p><hr>"
|
156 |
+
|
157 |
+
except Exception as e:
|
158 |
+
html += f"<p>Could not generate plots for {col}. Error: {e}</p><hr>"
|
159 |
+
|
160 |
+
return html
|
161 |
+
|
162 |
+
def analyze_univariate_categorical_html(df, categorical_cols):
|
163 |
+
"""Generates HTML for univariate analysis of categorical columns."""
|
164 |
+
html = "<h2>6. Univariate Analysis (Categorical Columns)</h2>"
|
165 |
+
html += "<p><i>Analyzing frequency distributions of individual categorical features using Count Plots.</i></p>"
|
166 |
+
if not categorical_cols:
|
167 |
+
html += "<p>No categorical/object columns found to analyze.</p>"
|
168 |
+
return html
|
169 |
+
|
170 |
+
plot_threshold = 50 # Max unique values for plotting
|
171 |
+
|
172 |
+
for col in categorical_cols:
|
173 |
+
html += f"<h3>Analyzing: '{col}'</h3>"
|
174 |
+
try:
|
175 |
+
unique_count = df[col].nunique()
|
176 |
+
html += f"<p><b>Number of Unique Values:</b> {unique_count}</p>"
|
177 |
+
|
178 |
+
if unique_count == 0:
|
179 |
+
html += "<p><i>Column has no values.</i></p><hr>"
|
180 |
+
continue
|
181 |
+
elif unique_count > plot_threshold:
|
182 |
+
html += f"<p><i>Skipping plot as unique value count ({unique_count}) exceeds threshold ({plot_threshold}). Showing Top 15 value counts instead.</i></p>"
|
183 |
+
top_15_counts = df[col].value_counts().head(15)
|
184 |
+
html += "<pre>" + top_15_counts.to_string() + "</pre><hr>"
|
185 |
+
else:
|
186 |
+
# Plot Count Plot
|
187 |
+
fig, ax = plt.subplots(figsize=(10, max(5, unique_count * 0.3))) # Adjust height
|
188 |
+
plot_order = df[col].value_counts().index
|
189 |
+
sns.countplot(y=df[col], order=plot_order, palette='viridis', ax=ax)
|
190 |
+
ax.set_title(f'Frequency Count of {col}')
|
191 |
+
ax.set_xlabel('Count')
|
192 |
+
ax.set_ylabel(col)
|
193 |
+
plt.tight_layout()
|
194 |
+
img_str = fig_to_base64(fig)
|
195 |
+
html += f'<img src="{img_str}" alt="Count Plot for {col}"><hr>'
|
196 |
+
|
197 |
+
except Exception as e:
|
198 |
+
html += f"<p>Could not generate plot/counts for {col}. Error: {e}</p><hr>"
|
199 |
+
|
200 |
+
return html
|
201 |
+
|
202 |
+
def analyze_bivariate_numerical_html(df, numerical_cols):
|
203 |
+
"""Generates HTML for bivariate analysis of numerical columns."""
|
204 |
+
html = "<h2>7. Bivariate Analysis (Numerical vs. Numerical)</h2>"
|
205 |
+
html += "<p><i>Analyzing relationships between pairs of numerical features using Correlation Matrix and Pair Plots.</i></p>"
|
206 |
+
|
207 |
+
if len(numerical_cols) < 2:
|
208 |
+
html += "<p>Need at least two numerical columns for this analysis.</p>"
|
209 |
+
return html
|
210 |
+
|
211 |
+
# Correlation Heatmap
|
212 |
+
html += "<h3>(a) Correlation Matrix Heatmap:</h3>"
|
213 |
+
try:
|
214 |
+
correlation_matrix = df[numerical_cols].corr()
|
215 |
+
fig, ax = plt.subplots(figsize=(12, 10))
|
216 |
+
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=".2f", linewidths=.5, ax=ax)
|
217 |
+
ax.set_title('Correlation Matrix of Numerical Features')
|
218 |
+
img_str = fig_to_base64(fig)
|
219 |
+
html += f'<img src="{img_str}" alt="Correlation Matrix"><br>'
|
220 |
+
html += "<p><i>Interpretation: Values close to +1 indicate strong positive linear correlation, close to -1 indicate strong negative linear correlation, close to 0 indicate weak or no linear correlation.</i></p>"
|
221 |
+
except Exception as e:
|
222 |
+
html += f"<p>Could not generate correlation heatmap. Error: {e}</p>"
|
223 |
+
|
224 |
+
# Pair Plot
|
225 |
+
pairplot_threshold = 7 # Limit features for pairplot
|
226 |
+
html += f"<h3>(b) Pair Plot (Threshold: {pairplot_threshold} features):</h3>"
|
227 |
+
if len(numerical_cols) <= pairplot_threshold:
|
228 |
+
html += f"<p><i>Generating Pair Plot for {len(numerical_cols)} numerical features... (May take a moment)</i></p>"
|
229 |
+
try:
|
230 |
+
pair_plot_fig = sns.pairplot(df[numerical_cols], diag_kind='kde')
|
231 |
+
pair_plot_fig.fig.suptitle('Pair Plot of Numerical Features', y=1.02) # Adjust title position
|
232 |
+
# Convert the PairGrid object's figure to base64
|
233 |
+
img_str = fig_to_base64(pair_plot_fig.fig)
|
234 |
+
html += f'<img src="{img_str}" alt="Pair Plot"><br>'
|
235 |
+
except Exception as e:
|
236 |
+
html += f"<p>Could not generate pair plot. Error: {e}</p>"
|
237 |
+
html += "<p><i>Pairplots can sometimes fail with certain data types or distributions, or if memory is limited.</i></p>"
|
238 |
+
else:
|
239 |
+
html += f"<p><i>Skipping Pair Plot because the number of numerical features ({len(numerical_cols)}) exceeds the threshold ({pairplot_threshold}).</i></p>"
|
240 |
+
|
241 |
+
return html
|
242 |
+
|
243 |
+
def analyze_bivariate_num_cat_html(df, numerical_cols, categorical_cols):
|
244 |
+
"""Generates HTML for bivariate analysis of numerical vs. categorical columns."""
|
245 |
+
html = "<h2>8. Bivariate Analysis (Numerical vs. Categorical)</h2>"
|
246 |
+
html += "<p><i>Analyzing distributions of numerical features across different categories using Box Plots.</i></p>"
|
247 |
+
|
248 |
+
if not numerical_cols or not categorical_cols:
|
249 |
+
html += "<p>Need both numerical and categorical columns for this analysis.</p>"
|
250 |
+
return html
|
251 |
+
|
252 |
+
cat_nunique_threshold = 20
|
253 |
+
cats_to_analyze = [col for col in categorical_cols if df[col].nunique() <= cat_nunique_threshold]
|
254 |
+
|
255 |
+
if not cats_to_analyze:
|
256 |
+
html += f"<p>No categorical columns with a reasonable number of unique values (<= {cat_nunique_threshold}) found for plotting against numerical features.</p>"
|
257 |
+
return html
|
258 |
+
|
259 |
+
html += f"<p><i>Analyzing numerical columns against these categorical columns (max {cat_nunique_threshold} unique values): <code>{cats_to_analyze}</code></i></p>"
|
260 |
+
|
261 |
+
for num_col in numerical_cols:
|
262 |
+
for cat_col in cats_to_analyze:
|
263 |
+
html += f"<h3>Analyzing: '{num_col}' vs '{cat_col}'</h3>"
|
264 |
+
try:
|
265 |
+
# Check if category column has data
|
266 |
+
if df[cat_col].isnull().all() or df[cat_col].nunique() == 0:
|
267 |
+
html += f"<p><i>Skipping plot: Categorical column '{cat_col}' has no valid data or only one unique value after dropping NaNs.</i></p><hr>"
|
268 |
+
continue
|
269 |
+
|
270 |
+
fig, ax = plt.subplots(figsize=(12, 6))
|
271 |
+
sns.boxplot(x=df[cat_col], y=df[num_col], palette='viridis', ax=ax, order=sorted(df[cat_col].dropna().unique())) # Added order and dropna
|
272 |
+
ax.set_title(f'Box Plot of {num_col} by {cat_col}')
|
273 |
+
ax.set_xlabel(cat_col)
|
274 |
+
ax.set_ylabel(num_col)
|
275 |
+
|
276 |
+
# Rotate x-axis labels if they are long or numerous
|
277 |
+
if df[cat_col].nunique() > 5:
|
278 |
+
plt.xticks(rotation=45, ha='right')
|
279 |
+
|
280 |
+
plt.tight_layout()
|
281 |
+
img_str = fig_to_base64(fig)
|
282 |
+
html += f'<img src="{img_str}" alt="Box plot of {num_col} by {cat_col}"><hr>'
|
283 |
+
|
284 |
+
except Exception as e:
|
285 |
+
html += f"<p>Could not generate box plot for '{num_col}' vs '{cat_col}'. Error: {e}</p><hr>"
|
286 |
+
|
287 |
+
return html
|
288 |
+
|
289 |
+
def get_analysis_summary_html(df, missing_table_html):
|
290 |
+
"""Generates HTML for the summary section."""
|
291 |
+
html = "<h2>9. Analysis Summary & Next Steps</h2>"
|
292 |
+
html += "<p>This automated analysis provided a first look at the dataset's structure, content, distributions, and basic relationships.</p>"
|
293 |
+
html += "<h3>Key Observations (Auto-Generated Summary):</h3>"
|
294 |
+
html += f"<ul><li>The dataset has <b>{df.shape[0]}</b> rows and <b>{df.shape[1]}</b> columns.</li>"
|
295 |
+
# Add more sophisticated summary points based on analysis if desired
|
296 |
+
if "Columns with Missing Values" in missing_table_html:
|
297 |
+
html += "<li>Missing values were detected (see Section 4 for details).</li>"
|
298 |
+
else:
|
299 |
+
html += "<li>No missing values were found.</li>"
|
300 |
+
html += "<li>Review the plots in Sections 5-8 for insights into distributions and relationships.</li>"
|
301 |
+
html += "<li><i>(Note: This is a basic summary. Customize with specific findings based on the generated report.)</i></li></ul>"
|
302 |
+
|
303 |
+
html += "<h3>Potential Next Steps:</h3>"
|
304 |
+
html += "<ol>"
|
305 |
+
html += "<li><b>Data Cleaning:</b> Address missing values (imputation/deletion), correct data types if needed, handle outliers (if appropriate).</li>"
|
306 |
+
html += "<li><b>Feature Engineering:</b> Create new features from existing ones (e.g., extracting date parts, combining categories).</li>"
|
307 |
+
html += "<li><b>Deeper Analysis:</b> Explore relationships further (statistical tests, different plots, multivariate analysis).</li>"
|
308 |
+
html += "<li><b>Domain-Specific Analysis:</b> Apply subject matter expertise for targeted questions.</li>"
|
309 |
+
html += "<li><b>Modeling:</b> Prepare data and build machine learning models if applicable.</li>"
|
310 |
+
html += "</ol>"
|
311 |
+
return html
|
312 |
+
|
313 |
+
def get_bonus_guide_html():
|
314 |
+
"""Generates HTML for the bonus guide."""
|
315 |
+
html = """
|
316 |
+
<h2>Bonus: How to Understand & Read Any Dataset</h2>
|
317 |
+
<p>Approaching a new dataset systematically:</p>
|
318 |
+
<ol>
|
319 |
+
<li><strong>Understand the Context:</strong> Source, purpose, data dictionary, timeframe.</li>
|
320 |
+
<li><strong>Load and Get a First Look:</strong> Use tools like pandas, check dimensions (`.shape`), peek at data (`.head()`, `.tail()`).</li>
|
321 |
+
<li><strong>Examine Metadata and Structure:</strong> Check column names (`.columns`), data types (`.info()`), memory usage. Correct types if necessary.</li>
|
322 |
+
<li><strong>Summarize the Data:</strong> Use `.describe()` for numerical (mean, median, std, min/max, quartiles) and categorical (unique count, top value, frequency) summaries. Check `.value_counts()` for specific categories.</li>
|
323 |
+
<li><strong>Handle Missing Data:</strong> Identify (`.isnull().sum()`) and quantify missing values. Decide on a strategy (deletion, imputation).</li>
|
324 |
+
<li><strong>Visualize (EDA):</strong>
|
325 |
+
<ul>
|
326 |
+
<li><em>Univariate:</em> Histograms, density plots, box plots (numerical); Count plots (categorical).</li>
|
327 |
+
<li><em>Bivariate:</em> Scatter plots, correlation matrix/heatmap (numerical vs. numerical); Box plots, violin plots (numerical vs. categorical); Crosstabs, stacked bars (categorical vs. categorical).</li>
|
328 |
+
<li><em>Multivariate:</em> Pair plots, faceting.</li>
|
329 |
+
</ul>
|
330 |
+
</li>
|
331 |
+
<li><strong>Ask Questions:</strong> Formulate specific questions based on context and initial findings.</li>
|
332 |
+
<li><strong>Iterate and Document:</strong> Data understanding is iterative. Document findings and decisions.</li>
|
333 |
+
</ol>
|
334 |
+
"""
|
335 |
+
return html
|
336 |
+
|
337 |
+
|
338 |
+
# --- Main Gradio Function ---
|
339 |
+
|
340 |
+
def generate_eda_report(uploaded_file):
|
341 |
+
"""
|
342 |
+
Main function called by Gradio. Takes an uploaded file, performs EDA,
|
343 |
+
and returns the path to a generated HTML report file.
|
344 |
+
"""
|
345 |
+
start_time = datetime.now()
|
346 |
+
if uploaded_file is None:
|
347 |
+
raise gr.Error("No file uploaded! Please upload a CSV file.")
|
348 |
+
|
349 |
+
try:
|
350 |
+
# Set visualization styles globally for the run
|
351 |
+
sns.set(style="whitegrid")
|
352 |
+
plt.rcParams['figure.figsize'] = (12, 6)
|
353 |
+
pd.set_option('display.max_columns', 50)
|
354 |
+
pd.set_option('display.float_format', lambda x: '%.2f' % x)
|
355 |
+
|
356 |
+
# Check file size (example: 100MB limit)
|
357 |
+
file_size_mb = os.path.getsize(uploaded_file.name) / (1024 * 1024)
|
358 |
+
if file_size_mb > 100:
|
359 |
+
raise gr.Error(f"File size ({file_size_mb:.2f} MB) exceeds the 100 MB limit.")
|
360 |
+
|
361 |
+
# Read the CSV file
|
362 |
+
# Use the temporary path provided by Gradio's File component
|
363 |
+
df = pd.read_csv(uploaded_file.name)
|
364 |
+
|
365 |
+
# Start building the HTML report
|
366 |
+
html_content = """
|
367 |
+
<!DOCTYPE html>
|
368 |
+
<html lang="en">
|
369 |
+
<head>
|
370 |
+
<meta charset="UTF-8">
|
371 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
372 |
+
<title>Automated EDA Report</title>
|
373 |
+
<style>
|
374 |
+
body { font-family: sans-serif; margin: 20px; }
|
375 |
+
h1, h2, h3 { color: #333; }
|
376 |
+
h1 { text-align: center; border-bottom: 2px solid #eee; padding-bottom: 10px; }
|
377 |
+
h2 { border-bottom: 1px solid #eee; padding-bottom: 5px; margin-top: 30px; }
|
378 |
+
h3 { margin-top: 20px; color: #555; }
|
379 |
+
table { border-collapse: collapse; width: auto; margin-top: 15px; margin-bottom: 15px; }
|
380 |
+
th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
|
381 |
+
th { background-color: #f2f2f2; }
|
382 |
+
tr:nth-child(even) { background-color: #f9f9f9; }
|
383 |
+
pre { background-color: #f5f5f5; padding: 10px; border: 1px solid #ccc; overflow-x: auto; }
|
384 |
+
code { background-color: #eee; padding: 2px 4px; border-radius: 3px; }
|
385 |
+
img { max-width: 100%; height: auto; display: block; margin: 15px auto; border: 1px solid #ddd; }
|
386 |
+
hr { border: 0; height: 1px; background: #ddd; margin: 30px 0; }
|
387 |
+
</style>
|
388 |
+
</head>
|
389 |
+
<body>
|
390 |
+
<h1>π Automated Data Explorer & Visualizer Report π</h1>
|
391 |
+
"""
|
392 |
+
report_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
393 |
+
html_content += f"<p style='text-align:center;'><i>Report generated on: {report_time}</i></p>"
|
394 |
+
html_content += f"<p style='text-align:center;'><i>Input file: {os.path.basename(uploaded_file.name)}</i></p>"
|
395 |
+
|
396 |
+
|
397 |
+
# --- Run EDA Steps ---
|
398 |
+
# 1. Initial Inspection
|
399 |
+
html_content += get_initial_inspection_html(df)
|
400 |
+
html_content += "<hr>"
|
401 |
+
|
402 |
+
# 2. Descriptive Statistics
|
403 |
+
html_content += get_descriptive_stats_html(df)
|
404 |
+
html_content += "<hr>"
|
405 |
+
|
406 |
+
# 3. Identify Column Types
|
407 |
+
col_types_html, num_cols, cat_cols = identify_column_types_html(df)
|
408 |
+
html_content += col_types_html
|
409 |
+
html_content += "<hr>"
|
410 |
+
|
411 |
+
# 4. Missing Values
|
412 |
+
missing_html = analyze_missing_values_html(df)
|
413 |
+
html_content += missing_html
|
414 |
+
html_content += "<hr>"
|
415 |
+
|
416 |
+
# 5. Univariate Numerical
|
417 |
+
html_content += analyze_univariate_numerical_html(df, num_cols)
|
418 |
+
html_content += "<hr>"
|
419 |
+
|
420 |
+
# 6. Univariate Categorical
|
421 |
+
html_content += analyze_univariate_categorical_html(df, cat_cols)
|
422 |
+
html_content += "<hr>"
|
423 |
+
|
424 |
+
# 7. Bivariate Numerical vs Numerical
|
425 |
+
html_content += analyze_bivariate_numerical_html(df, num_cols)
|
426 |
+
html_content += "<hr>"
|
427 |
+
|
428 |
+
# 8. Bivariate Numerical vs Categorical
|
429 |
+
html_content += analyze_bivariate_num_cat_html(df, num_cols, cat_cols)
|
430 |
+
html_content += "<hr>"
|
431 |
+
|
432 |
+
# 9. Summary
|
433 |
+
html_content += get_analysis_summary_html(df, missing_html) # Pass missing_html to check if missing values were found
|
434 |
+
html_content += "<hr>"
|
435 |
+
|
436 |
+
# 10. Bonus Guide
|
437 |
+
html_content += get_bonus_guide_html()
|
438 |
+
|
439 |
+
# --- Finalize HTML ---
|
440 |
+
html_content += f"<p style='text-align:center; margin-top: 30px;'><i>--- End of Report ---</i></p>"
|
441 |
+
end_time = datetime.now()
|
442 |
+
duration = end_time - start_time
|
443 |
+
html_content += f"<p style='text-align:center; font-size: small; color: grey;'><i>Analysis completed in {duration.total_seconds():.2f} seconds.</i></p>"
|
444 |
+
html_content += """
|
445 |
+
</body>
|
446 |
+
</html>
|
447 |
+
"""
|
448 |
+
|
449 |
+
# Save HTML content to a temporary file
|
450 |
+
# Use tempfile for better cross-platform compatibility and automatic cleanup
|
451 |
+
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=".html", encoding='utf-8') as temp_file:
|
452 |
+
temp_file.write(html_content)
|
453 |
+
report_path = temp_file.name # Get the path of the temp file
|
454 |
+
|
455 |
+
# Return the path to the generated HTML file for Gradio output
|
456 |
+
return report_path
|
457 |
+
|
458 |
+
except pd.errors.ParserError:
|
459 |
+
raise gr.Error("Error parsing CSV file. Please ensure it is a valid CSV format and delimiter is correctly inferred (usually comma).")
|
460 |
+
except FileNotFoundError:
|
461 |
+
raise gr.Error("Uploaded file not found. Please try uploading again.")
|
462 |
+
except ValueError as ve: # Catch specific value errors like Colab's upload error
|
463 |
+
raise gr.Error(f"Value Error: {ve}")
|
464 |
+
except Exception as e:
|
465 |
+
# Generic error catch - useful for debugging
|
466 |
+
import traceback
|
467 |
+
tb_str = traceback.format_exc()
|
468 |
+
print(f"An unexpected error occurred: {e}\n{tb_str}") # Log to console
|
469 |
+
raise gr.Error(f"An unexpected error occurred during analysis: {e}. Check console logs if running locally.")
|
470 |
+
|
471 |
+
|
472 |
+
# --- Gradio Interface Setup ---
|
473 |
+
|
474 |
+
description = """
|
475 |
+
**Effortless Dataset Insights π**
|
476 |
+
|
477 |
+
Upload your CSV dataset (max 100MB) and get an automated Exploratory Data Analysis (EDA) report.
|
478 |
+
The report includes:
|
479 |
+
1. Basic Info (Shape, Data Types, Head/Tail)
|
480 |
+
2. Descriptive Statistics
|
481 |
+
3. Missing Value Analysis & Heatmap
|
482 |
+
4. Univariate Analysis (Histograms, Box Plots, Count Plots)
|
483 |
+
5. Bivariate Analysis (Correlation Heatmap, Pair Plot [small datasets], Box Plots by Category)
|
484 |
+
6. Summary & Next Steps Guide
|
485 |
+
|
486 |
+
The output will be an HTML file that you can download and view in your browser.
|
487 |
+
"""
|
488 |
+
|
489 |
+
iface = gr.Interface(
|
490 |
+
fn=generate_eda_report,
|
491 |
+
inputs=gr.File(label="Upload CSV Dataset", file_types=[".csv"]),
|
492 |
+
outputs=gr.File(label="Download EDA Report (.html)"),
|
493 |
+
title="Effortless Dataset Insights",
|
494 |
+
description=description,
|
495 |
+
allow_flagging="never",
|
496 |
+
examples=[
|
497 |
+
# You can add paths to example CSV files here if you host them somewhere
|
498 |
+
# e.g., ["./examples/sample_data.csv"]
|
499 |
+
# Ensure these files exist if you uncomment this
|
500 |
+
],
|
501 |
+
theme=gr.themes.Soft() # Optional: Apply a theme
|
502 |
+
)
|
503 |
+
|
504 |
+
# --- Launch the App ---
|
505 |
+
if __name__ == "__main__":
|
506 |
+
iface.launch()
|