fix: remove module and put it back in app.py

app.py

@@ -3,6 +3,7 @@ import pandas as pd
 import gradio as gr
 import plotly.graph_objects as go
 import plotly.express as px
+from plotly.subplots import make_subplots
 from datetime import datetime, timedelta
 import json
 # Commenting out blockchain-related imports that cause loading issues
@@ -12,10 +13,589 @@ import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.dates as mdates
 import random
+import logging
+from typing import List, Dict, Any
 # Comment out the import for now and replace with dummy functions
 # from app_trans_new import create_transcation_visualizations,create_active_agents_visualizations
-# Import APR visualization functions from the new module
-from apr_visualization import generate_apr_visualizations
+# APR visualization functions integrated directly
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+# Global variable to store the data for reuse
+global_df = None
+
+# Configuration
+API_BASE_URL = "https://afmdb.autonolas.tech"
+
+def get_agent_type_by_name(type_name: str) -> Dict[str, Any]:
+    """Get agent type by name"""
+    response = requests.get(f"{API_BASE_URL}/api/agent-types/name/{type_name}")
+    if response.status_code == 404:
+        logger.error(f"Agent type '{type_name}' not found")
+        return None
+    response.raise_for_status()
+    return response.json()
+
+def get_attribute_definition_by_name(attr_name: str) -> Dict[str, Any]:
+    """Get attribute definition by name"""
+    response = requests.get(f"{API_BASE_URL}/api/attributes/name/{attr_name}")
+    if response.status_code == 404:
+        logger.error(f"Attribute definition '{attr_name}' not found")
+        return None
+    response.raise_for_status()
+    return response.json()
+
+def get_agents_by_type(type_id: int) -> List[Dict[str, Any]]:
+    """Get all agents of a specific type"""
+    response = requests.get(f"{API_BASE_URL}/api/agent-types/{type_id}/agents/")
+    if response.status_code == 404:
+        logger.error(f"No agents found for type ID {type_id}")
+        return []
+    response.raise_for_status()
+    return response.json()
+
+def get_attribute_values_by_type_and_attr(agents: List[Dict[str, Any]], attr_def_id: int) -> List[Dict[str, Any]]:
+    """Get all attribute values for a specific attribute definition across all agents of a given list"""
+    all_attributes = []
+    
+    # For each agent, get their attributes and filter for the one we want
+    for agent in agents:
+        agent_id = agent["agent_id"]
+        
+        # Call the /api/agents/{agent_id}/attributes/ endpoint
+        response = requests.get(f"{API_BASE_URL}/api/agents/{agent_id}/attributes/", params={"limit": 1000})
+        if response.status_code == 404:
+            logger.error(f"No attributes found for agent ID {agent_id}")
+            continue
+        
+        try:
+            response.raise_for_status()
+            agent_attrs = response.json()
+            
+            # Filter for the specific attribute definition ID
+            filtered_attrs = [attr for attr in agent_attrs if attr.get("attr_def_id") == attr_def_id]
+            all_attributes.extend(filtered_attrs)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error fetching attributes for agent ID {agent_id}: {e}")
+    
+    return all_attributes
+
+def get_agent_name(agent_id: int, agents: List[Dict[str, Any]]) -> str:
+    """Get agent name from agent ID"""
+    for agent in agents:
+        if agent["agent_id"] == agent_id:
+            return agent["agent_name"]
+    return "Unknown"
+
+def extract_apr_value(attr: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract APR value and timestamp from JSON value"""
+    try:
+        # The APR value is stored in the json_value field
+        if attr["json_value"] is None:
+            return {"apr": None, "timestamp": None, "agent_id": attr["agent_id"], "is_dummy": False}
+        
+        # If json_value is a string, parse it
+        if isinstance(attr["json_value"], str):
+            json_data = json.loads(attr["json_value"])
+        else:
+            json_data = attr["json_value"]
+        
+        apr = json_data.get("apr")
+        timestamp = json_data.get("timestamp")
+        
+        # Convert timestamp to datetime if it exists
+        timestamp_dt = None
+        if timestamp:
+            timestamp_dt = datetime.fromtimestamp(timestamp)
+            
+        return {"apr": apr, "timestamp": timestamp_dt, "agent_id": attr["agent_id"], "is_dummy": False}
+    except (json.JSONDecodeError, KeyError, TypeError) as e:
+        logger.error(f"Error parsing JSON value: {e} for agent_id: {attr.get('agent_id')}")
+        return {"apr": None, "timestamp": None, "agent_id": attr["agent_id"], "is_dummy": False}
+
+def fetch_apr_data_from_db():
+    """
+    Fetch APR data from database using the API.
+    """
+    global global_df
+    
+    try:
+        # Step 1: Find the Modius agent type
+        modius_type = get_agent_type_by_name("Modius")
+        if not modius_type:
+            logger.error("Modius agent type not found, using placeholder data")
+            global_df = pd.DataFrame([])
+            return global_df
+        
+        type_id = modius_type["type_id"]
+        
+        # Step 2: Find the APR attribute definition
+        apr_attr_def = get_attribute_definition_by_name("APR")
+        if not apr_attr_def:
+            logger.error("APR attribute definition not found, using placeholder data")
+            global_df = pd.DataFrame([])
+            return global_df
+            
+        attr_def_id = apr_attr_def["attr_def_id"]
+        
+        # Step 3: Get all agents of type Modius
+        modius_agents = get_agents_by_type(type_id)
+        if not modius_agents:
+            logger.error("No agents of type 'Modius' found")
+            global_df = pd.DataFrame([])
+            return global_df
+        
+        # Step 4: Fetch all APR values for Modius agents
+        apr_attributes = get_attribute_values_by_type_and_attr(modius_agents, attr_def_id)
+        if not apr_attributes:
+            logger.error("No APR values found for 'Modius' agents")
+            global_df = pd.DataFrame([])
+            return global_df
+        
+        # Step 5: Extract APR data
+        apr_data_list = []
+        for attr in apr_attributes:
+            apr_data = extract_apr_value(attr)
+            if apr_data["apr"] is not None and apr_data["timestamp"] is not None:
+                # Get agent name
+                agent_name = get_agent_name(attr["agent_id"], modius_agents)
+                # Add agent name to the data
+                apr_data["agent_name"] = agent_name
+                # Add is_dummy flag (all real data)
+                apr_data["is_dummy"] = False
+                
+                # Mark negative values as "Performance" metrics
+                if apr_data["apr"] < 0:
+                    apr_data["metric_type"] = "Performance"
+                else:
+                    apr_data["metric_type"] = "APR"
+                    
+                apr_data_list.append(apr_data)
+        
+        # Convert list of dictionaries to DataFrame
+        if not apr_data_list:
+            logger.error("No valid APR data extracted")
+            global_df = pd.DataFrame([])
+            return global_df
+        
+        global_df = pd.DataFrame(apr_data_list)
+        return global_df
+        
+    except requests.exceptions.RequestException as e:
+        logger.error(f"API request error: {e}")
+        global_df = pd.DataFrame([])
+        return global_df
+    except Exception as e:
+        logger.error(f"Error fetching APR data: {e}")
+        global_df = pd.DataFrame([])
+        return global_df
+
+def generate_apr_visualizations():
+    """Generate APR visualizations with real data only (no dummy data)"""
+    global global_df
+    
+    # Fetch data from database
+    df = fetch_apr_data_from_db()
+    
+    # If we got no data at all, return placeholder figures
+    if df.empty:
+        logger.info("No APR data available. Using fallback visualization.")
+        # Create empty visualizations with a message using Plotly
+        fig = go.Figure()
+        fig.add_annotation(
+            x=0.5, y=0.5,
+            text="No APR data available",
+            font=dict(size=20),
+            showarrow=False
+        )
+        fig.update_layout(
+            xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
+            yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
+        )
+        
+        # Save as static files for reference
+        fig.write_html("modius_apr_per_agent_graph.html")
+        fig.write_image("modius_apr_per_agent_graph.png")
+        fig.write_html("modius_apr_combined_graph.html")
+        fig.write_image("modius_apr_combined_graph.png")
+        
+        csv_file = None
+        return fig, fig, csv_file
+    
+    # No longer generating dummy data
+    # Set global_df for access by other functions
+    global_df = df
+    
+    # Save to CSV before creating visualizations
+    csv_file = save_to_csv(df)
+    
+    # Create per-agent time series graph (returns figure object)
+    per_agent_fig = create_time_series_graph_per_agent(df)
+    
+    # Create combined time series graph (returns figure object)
+    combined_fig = create_combined_time_series_graph(df)
+    
+    return per_agent_fig, combined_fig, csv_file
+
+def create_time_series_graph_per_agent(df):
+    """Create a time series graph for each agent using Plotly"""
+    # Get unique agents
+    unique_agents = df['agent_id'].unique()
+    
+    if len(unique_agents) == 0:
+        logger.error("No agent data to plot")
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No agent data available",
+            x=0.5, y=0.5,
+            showarrow=False, font=dict(size=20)
+        )
+        return fig
+    
+    # Create a subplot figure for each agent
+    fig = make_subplots(rows=len(unique_agents), cols=1, 
+                       subplot_titles=[f"Agent: {df[df['agent_id'] == agent_id]['agent_name'].iloc[0]}" 
+                                      for agent_id in unique_agents],
+                       vertical_spacing=0.1)
+    
+    # Plot data for each agent
+    for i, agent_id in enumerate(unique_agents):
+        agent_data = df[df['agent_id'] == agent_id].copy()
+        agent_name = agent_data['agent_name'].iloc[0]
+        row = i + 1
+        
+        # Add zero line to separate APR and Performance
+        fig.add_shape(
+            type="line", line=dict(dash="solid", width=1.5, color="black"),
+            y0=0, y1=0, x0=agent_data['timestamp'].min(), x1=agent_data['timestamp'].max(),
+            row=row, col=1
+        )
+        
+        # Add background colors
+        fig.add_shape(
+            type="rect", fillcolor="rgba(230, 243, 255, 0.3)", line=dict(width=0),
+            y0=0, y1=1000, x0=agent_data['timestamp'].min(), x1=agent_data['timestamp'].max(),
+            row=row, col=1, layer="below"
+        )
+        fig.add_shape(
+            type="rect", fillcolor="rgba(255, 230, 230, 0.3)", line=dict(width=0),
+            y0=-1000, y1=0, x0=agent_data['timestamp'].min(), x1=agent_data['timestamp'].max(),
+            row=row, col=1, layer="below"
+        )
+        
+        # Create separate dataframes for different data types
+        apr_data = agent_data[agent_data['metric_type'] == 'APR']
+        perf_data = agent_data[agent_data['metric_type'] == 'Performance']
+        
+        # Sort all data by timestamp for the line plots
+        combined_agent_data = agent_data.sort_values('timestamp')
+        
+        # Add main line connecting all points
+        fig.add_trace(
+            go.Scatter(
+                x=combined_agent_data['timestamp'], 
+                y=combined_agent_data['apr'],
+                mode='lines',
+                line=dict(color='purple', width=2),
+                name=f'{agent_name}',
+                legendgroup=agent_name,
+                showlegend=(i == 0),  # Only show in legend once
+                hovertemplate='Time: %{x}<br>Value: %{y:.2f}<extra></extra>'
+            ),
+            row=row, col=1
+        )
+        
+        # Add scatter points for APR values
+        if not apr_data.empty:
+            fig.add_trace(
+                go.Scatter(
+                    x=apr_data['timestamp'], 
+                    y=apr_data['apr'],
+                    mode='markers',
+                    marker=dict(color='blue', size=10, symbol='circle'),
+                    name='APR',
+                    legendgroup='APR',
+                    showlegend=(i == 0),
+                    hovertemplate='Time: %{x}<br>APR: %{y:.2f}<extra></extra>'
+                ),
+                row=row, col=1
+            )
+        
+        # Add scatter points for Performance values
+        if not perf_data.empty:
+            fig.add_trace(
+                go.Scatter(
+                    x=perf_data['timestamp'], 
+                    y=perf_data['apr'],
+                    mode='markers',
+                    marker=dict(color='red', size=10, symbol='square'),
+                    name='Performance',
+                    legendgroup='Performance',
+                    showlegend=(i == 0),
+                    hovertemplate='Time: %{x}<br>Performance: %{y:.2f}<extra></extra>'
+                ),
+                row=row, col=1
+            )
+        
+        # Update axes
+        fig.update_xaxes(title_text="Time", row=row, col=1)
+        fig.update_yaxes(title_text="Value", row=row, col=1, gridcolor='rgba(0,0,0,0.1)')
+    
+    # Update layout
+    fig.update_layout(
+        height=400 * len(unique_agents),
+        width=1000,
+        title_text="APR and Performance Values per Agent",
+        template="plotly_white",
+        legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=1.02,
+            xanchor="right",
+            x=1
+        ),
+        margin=dict(r=20, l=20, t=30, b=20),
+        hovermode="closest"
+    )
+    
+    # Save the figure (still useful for reference)
+    graph_file = "modius_apr_per_agent_graph.html"
+    fig.write_html(graph_file, include_plotlyjs='cdn', full_html=False)
+    
+    # Also save as image for compatibility
+    img_file = "modius_apr_per_agent_graph.png"
+    fig.write_image(img_file)
+    
+    logger.info(f"Per-agent graph saved to {graph_file} and {img_file}")
+    
+    # Return the figure object for direct use in Gradio
+    return fig
+
+def create_combined_time_series_graph(df):
+    """Create a combined time series graph for all agents using Plotly"""
+    if len(df) == 0:
+        logger.error("No data to plot combined graph")
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No data available",
+            x=0.5, y=0.5,
+            showarrow=False, font=dict(size=20)
+        )
+        return fig
+    
+    # Create Plotly figure
+    fig = go.Figure()
+    
+    # Get unique agents
+    unique_agents = df['agent_id'].unique()
+    
+    # Define a color scale for different agents
+    colors = px.colors.qualitative.Plotly[:len(unique_agents)]
+    
+    # Add background shapes for APR and Performance regions
+    min_time = df['timestamp'].min()
+    max_time = df['timestamp'].max()
+    
+    # Add shape for APR region (above zero)
+    fig.add_shape(
+        type="rect",
+        fillcolor="rgba(230, 243, 255, 0.3)",
+        line=dict(width=0),
+        y0=0, y1=1000,
+        x0=min_time, x1=max_time,
+        layer="below"
+    )
+    
+    # Add shape for Performance region (below zero)
+    fig.add_shape(
+        type="rect",
+        fillcolor="rgba(255, 230, 230, 0.3)",
+        line=dict(width=0),
+        y0=-1000, y1=0,
+        x0=min_time, x1=max_time,
+        layer="below"
+    )
+    
+    # Add zero line
+    fig.add_shape(
+        type="line",
+        line=dict(dash="solid", width=1.5, color="black"),
+        y0=0, y1=0,
+        x0=min_time, x1=max_time
+    )
+    
+    # Add data for each agent
+    for i, agent_id in enumerate(unique_agents):
+        agent_data = df[df['agent_id'] == agent_id].copy()
+        agent_name = agent_data['agent_name'].iloc[0]
+        color = colors[i % len(colors)]
+        
+        # Sort the data by timestamp
+        agent_data = agent_data.sort_values('timestamp')
+        
+        # Add the combined line for both APR and Performance
+        fig.add_trace(
+            go.Scatter(
+                x=agent_data['timestamp'],
+                y=agent_data['apr'],
+                mode='lines',
+                line=dict(color=color, width=2),
+                name=f'{agent_name}',
+                legendgroup=agent_name,
+                hovertemplate='Time: %{x}<br>Value: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>'
+            )
+        )
+        
+        # Add scatter points for APR values
+        apr_data = agent_data[agent_data['metric_type'] == 'APR']
+        if not apr_data.empty:
+            fig.add_trace(
+                go.Scatter(
+                    x=apr_data['timestamp'],
+                    y=apr_data['apr'],
+                    mode='markers',
+                    marker=dict(color=color, symbol='circle', size=8),
+                    name=f'{agent_name} APR',
+                    legendgroup=agent_name,
+                    showlegend=False,
+                    hovertemplate='Time: %{x}<br>APR: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>'
+                )
+            )
+        
+        # Add scatter points for Performance values
+        perf_data = agent_data[agent_data['metric_type'] == 'Performance']
+        if not perf_data.empty:
+            fig.add_trace(
+                go.Scatter(
+                    x=perf_data['timestamp'],
+                    y=perf_data['apr'],
+                    mode='markers',
+                    marker=dict(color=color, symbol='square', size=8),
+                    name=f'{agent_name} Perf',
+                    legendgroup=agent_name,
+                    showlegend=False,
+                    hovertemplate='Time: %{x}<br>Performance: %{y:.2f}<br>Agent: ' + agent_name + '<extra></extra>'
+                )
+            )
+    
+    # Update layout
+    fig.update_layout(
+        title="APR and Performance Values for All Agents",
+        xaxis_title="Time",
+        yaxis_title="Value",
+        template="plotly_white",
+        height=600,
+        width=1000,
+        legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=1.02,
+            xanchor="right",
+            x=1,
+            groupclick="toggleitem"
+        ),
+        margin=dict(r=20, l=20, t=30, b=20),
+        hovermode="closest"
+    )
+    
+    # Update axes
+    fig.update_xaxes(showgrid=True, gridwidth=1, gridcolor='rgba(0,0,0,0.1)')
+    fig.update_yaxes(showgrid=True, gridwidth=1, gridcolor='rgba(0,0,0,0.1)')
+    
+    # Save the figure (still useful for reference)
+    graph_file = "modius_apr_combined_graph.html"
+    fig.write_html(graph_file, include_plotlyjs='cdn', full_html=False)
+    
+    # Also save as image for compatibility
+    img_file = "modius_apr_combined_graph.png"
+    fig.write_image(img_file)
+    
+    logger.info(f"Combined graph saved to {graph_file} and {img_file}")
+    
+    # Return the figure object for direct use in Gradio
+    return fig
+
+def save_to_csv(df):
+    """Save the APR data DataFrame to a CSV file and return the file path"""
+    if df.empty:
+        logger.error("No APR data to save to CSV")
+        return None
+    
+    # Define the CSV file path
+    csv_file = "modius_apr_values.csv"
+    
+    # Save to CSV
+    df.to_csv(csv_file, index=False)
+    logger.info(f"APR data saved to {csv_file}")
+    
+    # Also generate a statistics CSV file
+    stats_df = generate_statistics_from_data(df)
+    stats_csv = "modius_apr_statistics.csv"
+    stats_df.to_csv(stats_csv, index=False)
+    logger.info(f"Statistics saved to {stats_csv}")
+    
+    return csv_file
+
+def generate_statistics_from_data(df):
+    """Generate statistics from the APR data"""
+    if df.empty:
+        return pd.DataFrame()
+    
+    # Get unique agents
+    unique_agents = df['agent_id'].unique()
+    stats_list = []
+    
+    # Generate per-agent statistics
+    for agent_id in unique_agents:
+        agent_data = df[df['agent_id'] == agent_id]
+        agent_name = agent_data['agent_name'].iloc[0]
+        
+        # APR statistics
+        apr_data = agent_data[agent_data['metric_type'] == 'APR']
+        real_apr = apr_data[apr_data['is_dummy'] == False]
+        
+        # Performance statistics
+        perf_data = agent_data[agent_data['metric_type'] == 'Performance']
+        real_perf = perf_data[perf_data['is_dummy'] == False]
+        
+        stats = {
+            'agent_id': agent_id,
+            'agent_name': agent_name,
+            'total_points': len(agent_data),
+            'apr_points': len(apr_data),
+            'performance_points': len(perf_data),
+            'real_apr_points': len(real_apr),
+            'real_performance_points': len(real_perf),
+            'avg_apr': apr_data['apr'].mean() if not apr_data.empty else None,
+            'avg_performance': perf_data['apr'].mean() if not perf_data.empty else None,
+            'max_apr': apr_data['apr'].max() if not apr_data.empty else None,
+            'min_apr': apr_data['apr'].min() if not apr_data.empty else None,
+            'latest_timestamp': agent_data['timestamp'].max().strftime('%Y-%m-%d %H:%M:%S') if not agent_data.empty else None
+        }
+        stats_list.append(stats)
+    
+    # Generate overall statistics
+    apr_only = df[df['metric_type'] == 'APR']
+    perf_only = df[df['metric_type'] == 'Performance']
+    
+    overall_stats = {
+        'agent_id': 'ALL',
+        'agent_name': 'All Agents',
+        'total_points': len(df),
+        'apr_points': len(apr_only),
+        'performance_points': len(perf_only),
+        'real_apr_points': len(apr_only[apr_only['is_dummy'] == False]),
+        'real_performance_points': len(perf_only[perf_only['is_dummy'] == False]),
+        'avg_apr': apr_only['apr'].mean() if not apr_only.empty else None,
+        'avg_performance': perf_only['apr'].mean() if not perf_only.empty else None,
+        'max_apr': apr_only['apr'].max() if not apr_only.empty else None,
+        'min_apr': apr_only['apr'].min() if not apr_only.empty else None,
+        'latest_timestamp': df['timestamp'].max().strftime('%Y-%m-%d %H:%M:%S') if not df.empty else None
+    }
+    stats_list.append(overall_stats)
+    
+    return pd.DataFrame(stats_list)
 
 # Create dummy functions for the commented out imports
 def create_transcation_visualizations():
@@ -390,4 +970,4 @@ def dashboard():
 
 # Launch the dashboard
 if __name__ == "__main__":
-    dashboard().launch()
+    dashboard().launch()