data saving

app.py

@@ -140,43 +140,33 @@ def save_dataset_to_hub(df: pd.DataFrame, dataset_name: str) -> tuple[bool, str]
             
         print(f"Saving {len(df)} steps to {dataset_name}...")
         
-        # Convert complex types to strings before creating dataset
-        for col in df.columns:
-            if df[col].dtype == 'object':
-                def safe_convert(x):
-                    if x is None or x == "None":
-                        return None
-                    if isinstance(x, (list, tuple, dict)):
-                        return str(x)
-                    return str(x) if pd.notna(x) else None
-                
-                df[col] = df[col].apply(safe_convert)
+        # Create a copy of the DataFrame to avoid modifying the original
+        df_to_save = df.copy()
+        
+        # Convert all columns to basic Python types
+        for col in df_to_save.columns:
+            def convert_to_basic_type(x):
+                if x is None or x == "None":
+                    return None
+                if isinstance(x, (list, tuple)):
+                    return [str(item) if item is not None else None for item in x]
+                if isinstance(x, dict):
+                    return {str(k): str(v) if v is not None else None for k, v in x.items()}
+                if hasattr(x, 'dict'):
+                    return x.dict()
+                if hasattr(x, '__dict__'):
+                    return str(x.__dict__)
+                return str(x) if pd.notna(x) else None
+            
+            df_to_save[col] = df_to_save[col].apply(convert_to_basic_type)
         
         # Convert to dataset
-        dataset = datasets.Dataset.from_pandas(df)
+        dataset = datasets.Dataset.from_pandas(df_to_save)
         
-        # Add metadata
+        # Add metadata with explicit string types for all columns
         dataset.info.description = "Agent steps data from evaluation run"
         dataset.info.features = {
-            'task_id': datasets.Value('string'),
-            'step_class': datasets.Value('string'),
-            'model_input_messages': datasets.Value('string'),
-            'tool_calls': datasets.Value('string'),
-            'start_time': datasets.Value('string'),
-            'end_time': datasets.Value('string'),
-            'step_number': datasets.Value('int64'),
-            'error': datasets.Value('string'),
-            'duration': datasets.Value('float64'),
-            'model_output_message': datasets.Value('string'),
-            'model_output': datasets.Value('string'),
-            'observations': datasets.Value('string'),
-            'observations_images': datasets.Value('string'),
-            'action_output': datasets.Value('string'),
-            'plan': datasets.Value('string'),
-            'task': datasets.Value('string'),
-            'task_images': datasets.Value('string'),
-            'system_prompt': datasets.Value('string'),
-            'final_answer': datasets.Value('string')
+            col: datasets.Value('string') for col in df_to_save.columns
         }
         
         # Save to hub with token
@@ -186,7 +176,7 @@ def save_dataset_to_hub(df: pd.DataFrame, dataset_name: str) -> tuple[bool, str]
             token=os.getenv("HUGGINGFACE_API_KEY")
         )
         
-        return True, f"Successfully saved {len(df)} steps to {dataset_name}"
+        return True, f"Successfully saved {len(df_to_save)} steps to {dataset_name}"
         
     except Exception as e:
         error_msg = f"Error saving dataset: {str(e)}"