major: prep implementtation of vLLM smart agents

app_mcp.py

@@ -19,7 +19,10 @@ from utils.minmax import preprocess as minmax_preprocess
 from utils.ela import genELA as ELA
 from utils.wavelet import wavelet_blocking_noise_estimation
 from utils.bitplane import bit_plane_extractor
-# from utils.exif import exif_full_dump / currently not working
+from utils.hf_logger import log_inference_data
+from utils.weight_management import ContextualWeightOverrideAgent, ModelWeightManager
+from utils.monitoring_agents import EnsembleMonitorAgent, WeightOptimizationAgent, SystemHealthAgent
+from utils.smart_agents import ContextualIntelligenceAgent, ForensicAnomalyDetectionAgent
 
 from forensics.registry import register_model, MODEL_REGISTRY, ModelEntry
 
@@ -302,170 +305,15 @@ def get_consensus_label(results):
 
 # Update predict_image_with_json to return consensus label
 
-class ModelWeightManager:
-    def __init__(self):
-        self.base_weights = {
-            "model_1": 0.15,  # SwinV2 Based
-            "model_2": 0.15,  # ViT Based
-            "model_3": 0.15,  # SDXL Dataset
-            "model_4": 0.15,  # SDXL + FLUX
-            "model_5": 0.15,  # ViT Based
-            "model_5b": 0.10, # ViT Based, Newer Dataset
-            "model_6": 0.10,  # Swin, Midj + SDXL
-            "model_7": 0.05   # ViT
-        }
-        self.situation_weights = {
-            "high_confidence": 1.2,    # Boost weights for high confidence predictions
-            "low_confidence": 0.8,     # Reduce weights for low confidence
-            "conflict": 0.5,          # Reduce weights when models disagree
-            "consensus": 1.5          # Boost weights when models agree
-        }
-    
-    def adjust_weights(self, predictions, confidence_scores):
-        """Dynamically adjust weights based on prediction patterns"""
-        adjusted_weights = self.base_weights.copy()
-        
-        # Check for consensus
-        if self._has_consensus(predictions):
-            for model in adjusted_weights:
-                adjusted_weights[model] *= self.situation_weights["consensus"]
-        
-        # Check for conflicts
-        if self._has_conflicts(predictions):
-            for model in adjusted_weights:
-                adjusted_weights[model] *= self.situation_weights["conflict"]
-        
-        # Adjust based on confidence
-        for model, confidence in confidence_scores.items():
-            if confidence > 0.8:
-                adjusted_weights[model] *= self.situation_weights["high_confidence"]
-            elif confidence < 0.5:
-                adjusted_weights[model] *= self.situation_weights["low_confidence"]
-        
-        return self._normalize_weights(adjusted_weights)
-    
-    def _has_consensus(self, predictions):
-        """Check if models agree on prediction"""
-        return len(set(predictions.values())) == 1
-    
-    def _has_conflicts(self, predictions):
-        """Check if models have conflicting predictions"""
-        return len(set(predictions.values())) > 2
-    
-    def _normalize_weights(self, weights):
-        """Normalize weights to sum to 1"""
-        total = sum(weights.values())
-        return {k: v/total for k, v in weights.items()}
-
-class EnsembleMonitorAgent:
-    def __init__(self):
-        self.performance_metrics = {
-            "model_accuracy": {},
-            "response_times": {},
-            "confidence_distribution": {},
-            "consensus_rate": 0.0
-        }
-        self.alerts = []
-    
-    def monitor_prediction(self, model_id, prediction, confidence, response_time):
-        """Monitor individual model performance"""
-        if model_id not in self.performance_metrics["model_accuracy"]:
-            self.performance_metrics["model_accuracy"][model_id] = []
-            self.performance_metrics["response_times"][model_id] = []
-            self.performance_metrics["confidence_distribution"][model_id] = []
-        
-        self.performance_metrics["response_times"][model_id].append(response_time)
-        self.performance_metrics["confidence_distribution"][model_id].append(confidence)
-        
-        # Check for performance issues
-        self._check_performance_issues(model_id)
-    
-    def _check_performance_issues(self, model_id):
-        """Check for any performance anomalies"""
-        response_times = self.performance_metrics["response_times"][model_id]
-        if len(response_times) > 10:
-            avg_time = sum(response_times[-10:]) / 10
-            if avg_time > 2.0:  # More than 2 seconds
-                self.alerts.append(f"High latency detected for {model_id}: {avg_time:.2f}s")
-
-class WeightOptimizationAgent:
-    def __init__(self, weight_manager):
-        self.weight_manager = weight_manager
-        self.performance_history = []
-        self.optimization_threshold = 0.1  # 10% performance change triggers optimization
-    
-    def analyze_performance(self, predictions, actual_results):
-        """Analyze model performance and suggest weight adjustments"""
-        # Placeholder for actual_results. In a real scenario, this would come from a validation set.
-        # For now, we'll just track predictions.
-        self.performance_history.append(predictions)
-        
-        if self._should_optimize():
-            self._optimize_weights()
-    
-    def _should_optimize(self):
-        """Determine if weights should be optimized"""
-        if len(self.performance_history) < 10:
-            return False
-        
-        # Placeholder for actual performance calculation
-        # For demonstration, let's say we optimize every 10 runs
-        return len(self.performance_history) % 10 == 0
-    
-    def _optimize_weights(self):
-        """Optimize model weights based on performance"""
-        logger.info("Optimizing model weights based on recent performance.")
-        # This is where more sophisticated optimization logic would go.
-        # For example, you could slightly adjust weights of models that consistently predict correctly.
-        pass
-
-class SystemHealthAgent:
-    def __init__(self):
-        self.health_metrics = {
-            "memory_usage": [],
-            "gpu_utilization": [],
-            "model_load_times": {},
-            "error_rates": {}
-        }
-    
-    def monitor_system_health(self):
-        """Monitor overall system health"""
-        self._check_memory_usage()
-        self._check_gpu_utilization()
-        # You might add _check_model_health() here later
-    
-    def _check_memory_usage(self):
-        """Monitor memory usage"""
-        try:
-            import psutil
-            memory = psutil.virtual_memory()
-            self.health_metrics["memory_usage"].append(memory.percent)
-            
-            if memory.percent > 90:
-                logger.warning(f"High memory usage detected: {memory.percent}%")
-        except ImportError:
-            logger.warning("psutil not installed. Cannot monitor memory usage.")
-    
-    def _check_gpu_utilization(self):
-        """Monitor GPU utilization if available"""
-        if torch.cuda.is_available():
-            try:
-                gpu_util = torch.cuda.memory_allocated() / torch.cuda.max_memory_allocated()
-                self.health_metrics["gpu_utilization"].append(gpu_util)
-                
-                if gpu_util > 0.9:
-                    logger.warning(f"High GPU utilization detected: {gpu_util*100:.2f}%")
-            except Exception as e:
-                logger.warning(f"Error monitoring GPU utilization: {e}")
-        else:
-            logger.info("CUDA not available. Skipping GPU utilization monitoring.")
-
 def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_degrees, noise_level, sharpen_strength):
     # Initialize agents
     monitor_agent = EnsembleMonitorAgent()
     weight_manager = ModelWeightManager()
     optimization_agent = WeightOptimizationAgent(weight_manager)
     health_agent = SystemHealthAgent()
+    # New smart agents
+    context_agent = ContextualIntelligenceAgent()
+    anomaly_agent = ForensicAnomalyDetectionAgent()
     
     # Monitor system health
     health_agent.monitor_system_health()
@@ -476,8 +324,8 @@ def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_d
         img_pil = img
     img_np_og = np.array(img)  # Convert PIL Image to NumPy array
 
-    # Get predictions with timing
-    model_predictions = {}
+    # 1. Get initial predictions from all models
+    model_predictions_raw = {}
     confidence_scores = {}
     results = [] # To store the results for the DataFrame
 
@@ -494,24 +342,34 @@ def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_d
             model_end - model_start
         )
         
-        model_predictions[model_id] = result["Label"]
+        model_predictions_raw[model_id] = result["Label"]
         confidence_scores[model_id] = max(result.get("AI Score", 0.0), result.get("Real Score", 0.0))
         results.append(result) # Add individual model result to the list
     
-    # Get adjusted weights
-    adjusted_weights = weight_manager.adjust_weights(model_predictions, confidence_scores)
-    
-    # Optimize weights if needed
-    optimization_agent.analyze_performance(model_predictions, None) # Placeholder for actual results
+    # 2. Infer context tags using ContextualIntelligenceAgent
+    image_data_for_context = {
+        "width": img.width,
+        "height": img.height,
+        "mode": img.mode,
+        # Add more features like EXIF data if exif_full_dump is used
+    }
+    detected_context_tags = context_agent.infer_context_tags(image_data_for_context, model_predictions_raw)
+    logger.info(f"Detected context tags: {detected_context_tags}")
+
+    # 3. Get adjusted weights, passing context tags
+    adjusted_weights = weight_manager.adjust_weights(model_predictions_raw, confidence_scores, context_tags=detected_context_tags)
     
-    # Calculate weighted consensus
+    # 4. Optimize weights if needed
+    # `final_prediction_label` is determined AFTER weighted consensus, so analyze_performance will be called later
+
+    # 5. Calculate weighted consensus
     weighted_predictions = {
         "AI": 0.0,
         "REAL": 0.0,
         "UNCERTAIN": 0.0
     }
     
-    for model_id, prediction in model_predictions.items():
+    for model_id, prediction in model_predictions_raw.items(): # Use raw predictions for weighting
         # Ensure the prediction label is valid for weighted_predictions
         if prediction in weighted_predictions:
             weighted_predictions[prediction] += adjusted_weights[model_id]
@@ -524,8 +382,11 @@ def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_d
         final_prediction_label = "AI"
     elif weighted_predictions["REAL"] > weighted_predictions["AI"] and weighted_predictions["REAL"] > weighted_predictions["UNCERTAIN"]:
         final_prediction_label = "REAL"
+
+    # Call analyze_performance after final_prediction_label is known
+    optimization_agent.analyze_performance(final_prediction_label, None)
     
-    # Rest of your existing code remains the same after this point
+    # 6. Perform forensic processing
     gradient_image = gradient_processing(img_np_og)  # Added gradient processing
     minmax_image = minmax_preprocess(img_np_og)  # Added MinMax processing
 
@@ -537,7 +398,24 @@ def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_d
     ela3 = ELA(img_np_og, quality=75, scale=75, contrast=25, linear=False, grayscale=False)
     
     forensics_images = [img_pil, ela1, ela2, ela3, gradient_image, minmax_image]
-    
+
+    # 7. Generate boilerplate descriptions for forensic outputs for anomaly agent
+    forensic_output_descriptions = [
+        f"Original augmented image (PIL): {img_pil.width}x{img_pil.height}",
+        "ELA analysis (Pass 1): Grayscale error map, quality 75.",
+        "ELA analysis (Pass 2): Grayscale error map, quality 75, enhanced contrast.",
+        "ELA analysis (Pass 3): Color error map, quality 75, enhanced contrast.",
+        "Gradient processing: Highlights edges and transitions.",
+        "MinMax processing: Deviations in local pixel values."
+    ]
+    # You could also add descriptions for Wavelet and Bit Plane if they were dynamic outputs
+    # For instance, if wavelet_blocking_noise_estimation had parameters that changed and you wanted to describe them.
+
+    # 8. Analyze forensic outputs for anomalies using ForensicAnomalyDetectionAgent
+    anomaly_detection_results = anomaly_agent.analyze_forensic_outputs(forensic_output_descriptions)
+    logger.info(f"Forensic anomaly detection: {anomaly_detection_results["summary"]}")
+
+
     # Prepare table rows for Dataframe (exclude model path)
     table_rows = [[
         r.get("Model", ""),
@@ -549,7 +427,54 @@ def predict_image_with_json(img, confidence_threshold, augment_methods, rotate_d
     
     # The get_consensus_label function is now replaced by final_prediction_label from weighted consensus
     consensus_html = f"<b><span style='color:{'red' if final_prediction_label == 'AI' else ('green' if final_prediction_label == 'REAL' else 'orange')}'>{final_prediction_label}</span></b>"
-    
+
+    # Prepare data for logging to Hugging Face dataset
+    inference_params = {
+        "confidence_threshold": confidence_threshold,
+        "augment_methods": augment_methods,
+        "rotate_degrees": rotate_degrees,
+        "noise_level": noise_level,
+        "sharpen_strength": sharpen_strength,
+        "detected_context_tags": detected_context_tags
+    }
+
+    ensemble_output_data = {
+        "final_prediction_label": final_prediction_label,
+        "weighted_predictions": weighted_predictions,
+        "adjusted_weights": adjusted_weights
+    }
+
+    # Collect agent monitoring data
+    agent_monitoring_data_log = {
+        "ensemble_monitor": {
+            "alerts": monitor_agent.alerts,
+            "performance_metrics": monitor_agent.performance_metrics
+        },
+        "weight_optimization": {
+            "prediction_history_length": len(optimization_agent.prediction_history),
+            # You might add a summary of recent accuracy here if _calculate_accuracy is exposed
+        },
+        "system_health": {
+            "memory_usage": health_agent.health_metrics["memory_usage"],
+            "gpu_utilization": health_agent.health_metrics["gpu_utilization"]
+        },
+        "context_intelligence": {
+            "detected_context_tags": detected_context_tags
+        },
+        "forensic_anomaly_detection": anomaly_detection_results
+    }
+
+    # Log the inference data
+    log_inference_data(
+        original_image=img, # Use the original uploaded image
+        inference_params=inference_params,
+        model_predictions=results, # This already contains detailed results for each model
+        ensemble_output=ensemble_output_data,
+        forensic_images=forensics_images, # This is the list of PIL images generated by forensic tools
+        agent_monitoring_data=agent_monitoring_data_log,
+        human_feedback=None # This can be populated later with human review data
+    )
+
     return img_pil, forensics_images, table_rows, results, consensus_html
 
 with gr.Blocks(css="#post-gallery { overflow: hidden !important;} .grid-wrap{ overflow-y: hidden !important;} .ms-gr-ant-welcome-icon{ height:unset !important;} .tabs{margin-top:10px;}") as demo:
@@ -615,7 +540,7 @@ with gr.Blocks(css="#post-gallery { overflow: hidden !important;} .grid-wrap{ ov
                     outputs=outputs
                 )
             with gr.Tab("🙈 Project Introduction"):
-                gr.Markdown("# AI Generated / Deepfake Detection Models Leaderboard: Soon™")
+                gr.Markdown(QUICK_INTRO)
                 
             with gr.Tab("👑 Community Forensics Preview"):
                 temp_space = gr.load("aiwithoutborders-xyz/OpenSight-Community-Forensics-Preview", src="spaces")

utils/smart_agents.py

@@ -0,0 +1,54 @@
+import logging
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+class ContextualIntelligenceAgent:
+    def __init__(self):
+        # In a real scenario, this would involve an LLM call or a sophisticated rule engine
+        pass
+
+    def infer_context_tags(self, image_data: dict, initial_predictions: dict) -> list[str]:
+        """Simulates an LLM inferring context tags based on image data and predictions."""
+        context_tags = []
+        
+        # Boilerplate logic: infer tags based on simple cues
+        if image_data.get("width", 0) > 1000 and image_data.get("height", 0) > 1000:
+            context_tags.append("high_resolution")
+        
+        # Example based on initial broad prediction (e.g., if any model strongly predicts 'real')
+        if any(v.get("Real Score", 0) > 0.9 for v in initial_predictions.values()):
+            context_tags.append("potentially_natural_scene")
+
+        # Mock external detection (e.g., from a simpler scene classification model or EXIF data)
+        # For demonstration, we'll hardcode some possible tags here.
+        # In a real system, you'd feed actual image features or metadata to an LLM.
+        mock_tags = ["outdoor", "sunny"] # These could be returned by an actual LLM based on input
+        for tag in mock_tags:
+            if tag not in context_tags:
+                context_tags.append(tag)
+
+        return context_tags
+
+class ForensicAnomalyDetectionAgent:
+    def __init__(self):
+        # In a real scenario, this would involve an LLM call to analyze textual descriptions
+        pass
+
+    def analyze_forensic_outputs(self, forensic_output_descriptions: list[str]) -> dict:
+        """Simulates an LLM analyzing descriptions of forensic images for anomalies."""
+        anomalies = {"summary": "No significant anomalies detected.", "details": []}
+
+        # Boilerplate logic: look for keywords in descriptions
+        for desc in forensic_output_descriptions:
+            if "strong edges" in desc.lower() and "ela" in desc.lower():
+                anomalies["summary"] = "Potential manipulation indicated by ELA."
+                anomalies["details"].append("ELA: Unusually strong edges detected, suggesting image compositing.")
+            if "unexpected patterns" in desc.lower() and "bit plane" in desc.lower():
+                anomalies["summary"] = "Anomalies detected in bit plane data."
+                anomalies["details"].append("Bit Plane: Irregular patterns found, possibly indicating hidden data or processing.")
+
+        if len(anomalies["details"]) > 0:
+            anomalies["summary"] = "Multiple anomalies detected across forensic outputs."
+            
+        return anomalies