Update app.py

app.py

@@ -1,4 +1,4 @@
-# app.py (Gaze Playback & Real-Time Prediction)
+# app.py 
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
@@ -20,16 +20,11 @@ plt.style.use('default')
 sns.set_palette("husl")
 
 class EnhancedAIvsRealGazeAnalyzer:
-    """
-    A comprehensive class to load, process, and analyze eye-tracking data.
-    It supports statistical analysis (RQ1), predictive modeling (RQ2),
-    and an innovative "Gaze Playback" dashboard.
-    """
     def __init__(self):
         self.questions = ['Q1', 'Q2', 'Q3', 'Q4', 'Q5', 'Q6']
         self.correct_answers = {'Pair1': 'B', 'Pair2': 'B', 'Pair3': 'B', 'Pair4': 'B', 'Pair5': 'B', 'Pair6': 'B'}
         self.combined_data = None
-        self.fixation_data = {}  # To store raw fixation data for each trial
+        self.fixation_data = {}
         self.participant_list = []
         self.model = None
         self.scaler = None
@@ -37,7 +32,6 @@ class EnhancedAIvsRealGazeAnalyzer:
         self.et_id_col = 'Participant name'
 
     def load_and_process_data(self, base_path, response_file):
-        """Loads aggregated metrics and raw fixation data, handling NaNs."""
         print("Loading and processing aggregated and raw fixation data...")
         self.response_data = pd.read_excel(response_file) if response_file.endswith('.xlsx') else pd.read_csv(response_file)
         self.response_data.columns = self.response_data.columns.str.strip()
@@ -48,27 +42,34 @@ class EnhancedAIvsRealGazeAnalyzer:
             file_path = f"{base_path}/Filtered_GenAI_Metrics_cleaned_{q}.xlsx"
             if os.path.exists(file_path):
                 xls = pd.ExcelFile(file_path)
-                # Load aggregated metrics (assuming it's the first sheet)
                 metrics_df = pd.read_excel(xls, sheet_name=0)
                 metrics_df['Question'] = q
                 all_metrics_dfs.append(metrics_df)
                 
-                # Load raw fixation data if the sheet exists
                 if 'Fixation-based AOI' in xls.sheet_names:
                     fix_df = pd.read_excel(xls, sheet_name='Fixation-based AOI')
                     fix_df['Question'] = q
-                    # ENSURE NO NULLS: Drop rows with invalid data crucial for playback
                     fix_df.dropna(subset=['Fixation point X', 'Fixation point Y', 'Gaze event duration (ms)'], inplace=True)
-                    self.et_id_col = next((c for c in fix_df.columns if 'participant' in c.lower()), 'Participant name')
-                    for participant, group in fix_df.groupby(self.et_id_col):
-                        self.fixation_data[(str(participant), q)] = group.reset_index(drop=True)
+                    # Use a local variable here to avoid confusion
+                    fix_et_id_col = next((c for c in fix_df.columns if 'participant' in c.lower()), None)
+                    if fix_et_id_col:
+                        for participant, group in fix_df.groupby(fix_et_id_col):
+                            self.fixation_data[(str(participant), q)] = group.reset_index(drop=True)
         
         if not all_metrics_dfs: raise ValueError("No aggregated metrics files were found.")
         self.combined_data = pd.concat(all_metrics_dfs, ignore_index=True)
         self.combined_data.columns = self.combined_data.columns.str.strip()
         
-        # Merge response data with aggregated metrics
-        resp_id_col = next((c for c in self.response_data.columns if 'participant' in c.lower()), 'Participant name')
+        # --- THIS IS THE KEY FIX ---
+        # 1. Dynamically find the participant ID column in the COMBINED metrics data.
+        self.et_id_col = next((c for c in self.combined_data.columns if 'participant' in c.lower()), None)
+        if not self.et_id_col: raise KeyError("Could not find a 'participant' column in the aggregated metrics data.")
+
+        # 2. Dynamically find the participant ID column in the RESPONSE data.
+        resp_id_col = next((c for c in self.response_data.columns if 'participant' in c.lower()), None)
+        if not resp_id_col: raise KeyError("Could not find a 'participant' column in the response sheet.")
+        # --- END OF FIX ---
+
         for pair, ans in self.correct_answers.items():
             if pair in self.response_data.columns:
                 self.response_data[f'{pair}_Correct'] = (self.response_data[pair].astype(str).str.strip().str.upper() == ans)
@@ -79,12 +80,13 @@ class EnhancedAIvsRealGazeAnalyzer:
         response_long = response_long.merge(correctness_long[[resp_id_col, 'Pair', 'Correct']], on=[resp_id_col, 'Pair'])
         q_to_pair = {f'Q{i+1}': f'Pair{i+1}' for i in range(6)}
         self.combined_data['Pair'] = self.combined_data['Question'].map(q_to_pair)
+        
+        # 3. Perform the merge using the correctly identified column names.
         self.combined_data = self.combined_data.merge(response_long, left_on=[self.et_id_col, 'Pair'], right_on=[resp_id_col, 'Pair'], how='left')
         self.combined_data['Answer_Correctness'] = self.combined_data['Correct'].map({True: 'Correct', False: 'Incorrect'})
         
         self.numeric_cols = self.combined_data.select_dtypes(include=np.number).columns.tolist()
         self.time_metrics = [c for c in self.numeric_cols if any(k in c.lower() for k in ['time', 'duration', 'fixation'])]
-        # Convert all participant IDs to strings to prevent sorting errors
         self.participant_list = sorted([str(p) for p in self.combined_data[self.et_id_col].unique()])
         print("Data loading complete.")
         return self
@@ -119,42 +121,30 @@ class EnhancedAIvsRealGazeAnalyzer:
         return summary_md, report_df, fig
     
     def _recalculate_features_from_fixations(self, fixations_df):
-        """Helper to dynamically create a feature vector from a list of fixations."""
         feature_vector = pd.Series(0.0, index=self.feature_names)
-        if fixations_df.empty:
-            return feature_vector.values.reshape(1, -1)
-        
-        # Recalculate duration-based features
+        if fixations_df.empty: return feature_vector.fillna(0).values.reshape(1, -1)
         if 'AOI name' in fixations_df.columns:
             for aoi_name, group in fixations_df.groupby('AOI name'):
                 col_name = f'Total fixation duration on {aoi_name}'
                 if col_name in feature_vector.index:
                     feature_vector[col_name] = group['Gaze event duration (ms)'].sum()
-        
         feature_vector['Total Recording Duration'] = fixations_df['Gaze event duration (ms)'].sum()
-        
         return feature_vector.fillna(0).values.reshape(1, -1)
 
     def generate_gaze_playback(self, participant, question, fixation_num):
-        """Generates the gaze playback and real-time prediction dashboard."""
         trial_key = (str(participant), question)
         if not participant or not question or trial_key not in self.fixation_data:
             return "Please select a valid trial with fixation data.", None, gr.Slider(interactive=False)
-
         all_fixations = self.fixation_data[trial_key]
         fixation_num = int(fixation_num)
-        
         slider_max = len(all_fixations)
         if fixation_num > slider_max: fixation_num = slider_max
         current_fixations = all_fixations.iloc[:fixation_num]
-        
         partial_features = self._recalculate_features_from_fixations(current_fixations)
         prediction_prob = self.model.predict_proba(self.scaler.transform(partial_features))[0]
         prob_correct = prediction_prob[1]
-
         fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 8), gridspec_kw={'height_ratios': [4, 1]})
         fig.suptitle(f"Gaze Playback for {participant} - {question}", fontsize=16, weight='bold')
-
         ax1.set_title(f"Displaying Fixations 1 through {fixation_num}/{slider_max}")
         ax1.set_xlim(0, 1920); ax1.set_ylim(1080, 0)
         ax1.set_aspect('equal'); ax1.tick_params(left=False, right=False, bottom=False, top=False, labelleft=False, labelbottom=False)
@@ -162,24 +152,19 @@ class EnhancedAIvsRealGazeAnalyzer:
         ax1.add_patch(patches.Rectangle((1920/2, 0), 1920/2, 1080, facecolor='blue', alpha=0.05))
         ax1.text(1920*0.25, 50, "Image A", ha='center', fontsize=14, alpha=0.5)
         ax1.text(1920*0.75, 50, "Image B", ha='center', fontsize=14, alpha=0.5)
-
         if not current_fixations.empty:
             points = current_fixations[['Fixation point X', 'Fixation point Y']]
             ax1.plot(points['Fixation point X'], points['Fixation point Y'], marker='o', color='grey', alpha=0.5, linestyle='-')
             ax1.scatter(points.iloc[-1]['Fixation point X'], points.iloc[-1]['Fixation point Y'], s=150, c='red', zorder=10, edgecolors='black')
-
         ax2.set_xlim(0, 1); ax2.set_yticks([])
         ax2.set_title("Live Prediction Confidence (Answer is 'Correct')")
         bar_color = 'green' if prob_correct > 0.5 else 'red'
         ax2.barh([0], [prob_correct], color=bar_color, height=0.5)
         ax2.axvline(0.5, color='black', linestyle='--', linewidth=1)
         ax2.text(prob_correct, 0, f" {prob_correct:.1%} ", va='center', ha='left' if prob_correct < 0.9 else 'right', color='white', weight='bold')
-
         plt.tight_layout(rect=[0, 0, 1, 0.95])
-        
         trial_info = self.combined_data[(self.combined_data[self.et_id_col].astype(str) == str(participant)) & (self.combined_data['Question'] == question)].iloc[0]
         summary_text = f"**Actual Answer:** `{trial_info['Answer_Correctness']}`"
-        
         return summary_text, fig, gr.Slider(maximum=slider_max, value=fixation_num, interactive=True)
 
 # --- DATA SETUP & GRADIO APP ---
@@ -205,7 +190,6 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                     rq1_summary_output=gr.Markdown(label="Statistical Summary")
                 with gr.Column(scale=2):
                     rq1_plot_output=gr.Plot(label="Metric Comparison")
-        
         with gr.TabItem("🤖 RQ2: Predicting Correctness from Gaze"):
             with gr.Row():
                 with gr.Column(scale=1):
@@ -216,7 +200,6 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                     rq2_summary_output=gr.Markdown(label="Model Performance Summary")
                     rq2_table_output=gr.Dataframe(label="Classification Report", interactive=False)
                     rq2_plot_output=gr.Plot(label="Feature Importance")
-
         with gr.TabItem("👁️ Gaze Playback & Real-Time Prediction"):
             gr.Markdown("### See the Prediction Evolve with Every Glance!")
             with gr.Row():
@@ -231,13 +214,10 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
 
     outputs_rq2 = [rq2_summary_output, rq2_table_output, rq2_plot_output]
     outputs_playback = [playback_summary, playback_plot, playback_slider]
-    
     rq1_metric_dropdown.change(fn=analyzer.analyze_rq1_metric, inputs=rq1_metric_dropdown, outputs=[rq1_plot_output, rq1_summary_output])
     rq2_test_size_slider.release(fn=analyzer.run_prediction_model, inputs=[rq2_test_size_slider, rq2_estimators_slider], outputs=outputs_rq2)
     rq2_estimators_slider.release(fn=analyzer.run_prediction_model, inputs=[rq2_test_size_slider, rq2_estimators_slider], outputs=outputs_rq2)
-    
     playback_inputs = [playback_participant, playback_question, playback_slider]
-    # Reset slider to 0 when a new trial is selected
     playback_participant.change(lambda: 0, None, playback_slider).then(fn=analyzer.generate_gaze_playback, inputs=playback_inputs, outputs=outputs_playback)
     playback_question.change(lambda: 0, None, playback_slider).then(fn=analyzer.generate_gaze_playback, inputs=playback_inputs, outputs=outputs_playback)
     playback_slider.release(fn=analyzer.generate_gaze_playback, inputs=playback_inputs, outputs=outputs_playback)