import os import numpy as np import torch import cv2 from PIL import Image import tempfile import uuid from typing import Dict, List, Any, Optional, Tuple from detection_model import DetectionModel from color_mapper import ColorMapper from visualization_helper import VisualizationHelper from evaluation_metrics import EvaluationMetrics from lighting_analyzer import LightingAnalyzer from scene_analyzer import SceneAnalyzer from places365_model import Places365Model class ImageProcessor: """ Class for handling image processing and object detection operations Separates processing logic from UI components """ def __init__(self, use_llm=True, llm_model_path=None, enable_places365=True, places365_model_name='resnet50_places365'): """Initialize the image processor with required components""" print(f"Initializing ImageProcessor with use_llm={use_llm}, enable_places365={enable_places365}") try: # Initialize basic components first self.use_llm = use_llm self.llm_model_path = llm_model_path self.enable_places365 = enable_places365 self.model_instances = {} # Initialize ColorMapper self.color_mapper = ColorMapper() print("ColorMapper initialized successfully") # Initialize LightingAnalyzer self.lighting_analyzer = LightingAnalyzer() print("LightingAnalyzer initialized successfully") # Initialize Places365 model if enabled self.places365_model = None if self.enable_places365: try: self.places365_model = Places365Model( model_name=places365_model_name, device=None ) print(f"Places365 model initialized successfully with {places365_model_name}") except Exception as e: print(f"Warning: Failed to initialize Places365 model: {e}") print("Continuing without Places365 analysis") self.enable_places365 = False self.places365_model = None # Initialize SceneAnalyzer with error handling self.scene_analyzer = None self.class_names = None # Will be set when first model is loaded try: # Initialize SceneAnalyzer without class_names (will be set later) self.scene_analyzer = SceneAnalyzer( class_names=None, use_llm=self.use_llm, use_clip=True, enable_landmark=True, llm_model_path=self.llm_model_path ) print("SceneAnalyzer initialized successfully") # Verify critical components if self.scene_analyzer is not None: print(f"SceneAnalyzer status - spatial_analyzer: {hasattr(self.scene_analyzer, 'spatial_analyzer')}, " f"descriptor: {hasattr(self.scene_analyzer, 'descriptor')}, " f"scene_describer: {hasattr(self.scene_analyzer, 'scene_describer')}") else: print("WARNING: scene_analyzer is None after initialization") except Exception as e: print(f"Error initializing SceneAnalyzer: {e}") import traceback traceback.print_exc() self.scene_analyzer = None print("ImageProcessor initialization completed successfully") except Exception as e: print(f"Critical error during ImageProcessor initialization: {e}") import traceback traceback.print_exc() raise RuntimeError(f"Failed to initialize ImageProcessor: {str(e)}") def get_model_instance(self, model_name: str, confidence: float = 0.25, iou: float = 0.25) -> DetectionModel: """ Get or create a model instance based on model name Args: model_name: Name of the model to use confidence: Confidence threshold for detection iou: IoU threshold for non-maximum suppression Returns: DetectionModel instance """ if model_name not in self.model_instances: print(f"Creating new model instance for {model_name}") self.model_instances[model_name] = DetectionModel( model_name=model_name, confidence=confidence, iou=iou ) else: print(f"Using existing model instance for {model_name}") self.model_instances[model_name].confidence = confidence return self.model_instances[model_name] def analyze_scene(self, detection_result: Any, lighting_info: Optional[Dict] = None, enable_landmark=True, places365_info=None) -> Dict: """ Perform scene analysis on detection results Args: detection_result: Object detection result from YOLOv8 lighting_info: Lighting condition analysis results (optional) enable_landmark: Whether to enable landmark detection places365_info: Places365 analysis results (optional) Returns: Dictionary containing scene analysis results """ print(f"DEBUG: analyze_scene received enable_landmark={enable_landmark}") try: # Check if detection_result has valid names class_names = getattr(detection_result, 'names', None) if detection_result else None # Initialize or reinitialize scene analyzer if needed if self.scene_analyzer is None: print("Scene analyzer not initialized, creating new instance") self.scene_analyzer = SceneAnalyzer( class_names=class_names, use_llm=self.use_llm, use_clip=True, enable_landmark=enable_landmark, llm_model_path=self.llm_model_path ) if self.scene_analyzer is None: raise ValueError("Failed to create SceneAnalyzer instance") else: # Update existing scene analyzer settings self.scene_analyzer.enable_landmark = enable_landmark # Update class names if available and different if class_names and self.scene_analyzer.class_names != class_names: self.scene_analyzer.class_names = class_names if hasattr(self.scene_analyzer, 'spatial_analyzer') and self.scene_analyzer.spatial_analyzer: self.scene_analyzer.spatial_analyzer.class_names = class_names # Update landmark detection settings in child components if hasattr(self.scene_analyzer, 'spatial_analyzer') and self.scene_analyzer.spatial_analyzer: self.scene_analyzer.spatial_analyzer.enable_landmark = enable_landmark # Perform scene analysis with lighting info and Places365 context scene_analysis = self.scene_analyzer.analyze( detection_result=detection_result, lighting_info=lighting_info, class_confidence_threshold=0.35, scene_confidence_threshold=0.6, enable_landmark=enable_landmark, places365_info=places365_info ) return scene_analysis except Exception as e: print(f"Error in scene analysis: {str(e)}") import traceback traceback.print_exc() # Return a valid default result return { "scene_type": "unknown", "confidence": 0.0, "description": f"Error during scene analysis: {str(e)}", "enhanced_description": "Scene analysis could not be completed due to an error.", "objects_present": [], "object_count": 0, "regions": {}, "possible_activities": [], "safety_concerns": [], "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0} } def analyze_lighting_conditions(self, image, places365_info: Optional[Dict] = None): """ 分析光照條件並考慮 Places365 場景資訊。 Args: image: 輸入圖像 places365_info: Places365 場景分析結果，用於覆蓋邏輯 Returns: Dict: 光照分析結果 """ return self.lighting_analyzer.analyze(image, places365_info=places365_info) def analyze_places365_scene(self, image): """ Analyze scene using Places365 model. Args: image: Input image (PIL Image) Returns: Dict: Places365 analysis results or None if disabled/failed """ if not self.enable_places365 or self.places365_model is None: return None try: if not isinstance(image, Image.Image): if isinstance(image, np.ndarray): image = Image.fromarray(image) else: print(f"Warning: Cannot process image of type {type(image)} for Places365") return None places365_result = self.places365_model.predict(image) if places365_result and places365_result.get('confidence', 0) > 0.1: print(f"Places365 detected: {places365_result['scene_label']} " f"(mapped: {places365_result['mapped_scene_type']}) " f"confidence: {places365_result['confidence']:.3f}") return places365_result else: print("Places365 analysis failed or low confidence") return None except Exception as e: print(f"Error in Places365 analysis: {str(e)}") return None def process_image(self, image: Any, model_name: str, confidence_threshold: float, filter_classes: Optional[List[int]] = None, enable_landmark: bool = True) -> Tuple[Any, str, Dict]: """ Process an image for object detection and scene analysis. Args: image: Input image (numpy array or PIL Image). model_name: Name of the model to use. confidence_threshold: Confidence threshold for detection. filter_classes: Optional list of classes to filter results. enable_landmark: Whether to enable landmark detection for this run. Returns: Tuple of (result_image_pil, result_text, stats_data_with_scene_analysis). """ model_instance = self.get_model_instance(model_name, confidence_threshold) if model_instance is None: return None, f"Failed to load model: {model_name}. Please check model configuration.", {} result = None stats_data = {} temp_path = None pil_image_for_processing = None # Use this to store the consistently processed PIL image try: if isinstance(image, np.ndarray): if image.ndim == 3 and image.shape[2] == 3: # RGB or BGR # Assuming BGR from OpenCV, convert to RGB for PIL standard image_rgb_np = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) pil_image_for_processing = Image.fromarray(image_rgb_np) elif image.ndim == 3 and image.shape[2] == 4: # RGBA or BGRA image_rgba_np = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA) # Ensure RGBA pil_image_for_processing = Image.fromarray(image_rgba_np).convert("RGB") # Convert to RGB elif image.ndim == 2: # Grayscale pil_image_for_processing = Image.fromarray(image).convert("RGB") else: pil_image_for_processing = Image.fromarray(image) # Hope for the best elif isinstance(image, Image.Image): pil_image_for_processing = image.copy() # Use a copy elif image is None: return None, "No image provided. Please upload an image.", {} else: return None, f"Unsupported image type: {type(image)}. Please provide a NumPy array or PIL Image.", {} if pil_image_for_processing.mode != "RGB": # Ensure final image is RGB pil_image_for_processing = pil_image_for_processing.convert("RGB") # Add Places365 scene analysis parallel to lighting analysis places365_info = self.analyze_places365_scene(pil_image_for_processing) lighting_info = self.analyze_lighting_conditions(pil_image_for_processing, places365_info=places365_info) temp_dir = tempfile.gettempdir() temp_filename = f"temp_{uuid.uuid4().hex}.jpg" temp_path = os.path.join(temp_dir, temp_filename) pil_image_for_processing.save(temp_path, format="JPEG") result = model_instance.detect(temp_path) if result is None or not hasattr(result, 'boxes'): scene_analysis_no_yolo = self.analyze_scene(result, lighting_info, enable_landmark=enable_landmark, places365_info=places365_info) desc_no_yolo = scene_analysis_no_yolo.get("enhanced_description", scene_analysis_no_yolo.get("description", "Detection failed, scene context analysis attempted.")) stats_data["scene_analysis"] = scene_analysis_no_yolo if places365_info: stats_data["places365_analysis"] = places365_info return pil_image_for_processing, desc_no_yolo, stats_data # 統計資訊 stats_data = EvaluationMetrics.calculate_basic_stats(result) spatial_metrics = EvaluationMetrics.calculate_distance_metrics(result) stats_data["spatial_metrics"] = spatial_metrics stats_data["lighting_conditions"] = lighting_info if places365_info: stats_data["places365_analysis"] = places365_info if filter_classes and len(filter_classes) > 0: classes = result.boxes.cls.cpu().numpy().astype(int) confs = result.boxes.conf.cpu().numpy() mask = np.isin(classes, filter_classes) filtered_stats_data = { "total_objects": int(np.sum(mask)), "class_statistics": {}, "average_confidence": float(np.mean(confs[mask])) if np.any(mask) else 0.0, "spatial_metrics": stats_data.get("spatial_metrics",{}), "lighting_conditions": lighting_info } if places365_info: filtered_stats_data["places365_analysis"] = places365_info names = result.names class_conf_sums = {} for cls_id_int, conf_val in zip(classes[mask], confs[mask]): cls_name = names[cls_id_int] if cls_name not in filtered_stats_data["class_statistics"]: filtered_stats_data["class_statistics"][cls_name] = {"count": 0} class_conf_sums[cls_name] = 0.0 filtered_stats_data["class_statistics"][cls_name]["count"] += 1 # 累計統計資訊 class_conf_sums[cls_name] += conf_val for cls_name_stat, data_stat in filtered_stats_data["class_statistics"].items(): data_stat["average_confidence"] = round(class_conf_sums[cls_name_stat] / data_stat["count"] if data_stat["count"] > 0 else 0.0, 4) stats_data = filtered_stats_data viz_data = EvaluationMetrics.generate_visualization_data(result, self.color_mapper.get_all_colors()) result_image_pil = VisualizationHelper.visualize_detection( temp_path, result, color_mapper=self.color_mapper, figsize=(12, 12), return_pil=True, filter_classes=filter_classes ) result_text_summary = EvaluationMetrics.format_detection_summary(viz_data) # Pass the enable_landmark parameter from function signature # Initialize or update scene analyzer if needed if self.scene_analyzer is None: print("Creating SceneAnalyzer in process_image") self.scene_analyzer = SceneAnalyzer( class_names=result.names if result else None, use_llm=self.use_llm, use_clip=True, enable_landmark=enable_landmark, llm_model_path=self.llm_model_path ) if self.scene_analyzer is None: print("ERROR: Failed to create SceneAnalyzer in process_image") else: # Update existing scene analyzer with current settings if result and hasattr(result, 'names'): self.scene_analyzer.class_names = result.names if hasattr(self.scene_analyzer, 'spatial_analyzer') and self.scene_analyzer.spatial_analyzer: self.scene_analyzer.spatial_analyzer.class_names = result.names self.scene_analyzer.enable_landmark = enable_landmark if hasattr(self.scene_analyzer, 'spatial_analyzer') and self.scene_analyzer.spatial_analyzer: self.scene_analyzer.spatial_analyzer.enable_landmark = enable_landmark # Perform scene analysis using the existing analyze_scene method scene_analysis_result = self.analyze_scene( detection_result=result, lighting_info=lighting_info, enable_landmark=enable_landmark, places365_info=places365_info ) stats_data["scene_analysis"] = scene_analysis_result final_result_text = result_text_summary # Use enable_landmark parameter for landmark block if enable_landmark and "detected_landmarks" in scene_analysis_result: landmarks_detected = scene_analysis_result.get("detected_landmarks", []) if not landmarks_detected and scene_analysis_result.get("primary_landmark"): primary_lm = scene_analysis_result.get("primary_landmark") if isinstance(primary_lm, dict): landmarks_detected = [primary_lm] if landmarks_detected: final_result_text += "\n\n--- Detected Landmarks ---\n" # Ensure drawing on the correct PIL image img_to_draw_on = result_image_pil.copy() # Draw on a copy img_for_drawing_cv2 = cv2.cvtColor(np.array(img_to_draw_on), cv2.COLOR_RGB2BGR) for landmark_item in landmarks_detected: if not isinstance(landmark_item, dict): continue # Use .get() for all potentially missing keys 比較保險 landmark_name_disp = landmark_item.get("class_name", landmark_item.get("name", "N/A")) landmark_loc_disp = landmark_item.get("location", "N/A") landmark_conf_disp = landmark_item.get("confidence", 0.0) final_result_text += f"• {landmark_name_disp} ({landmark_loc_disp}, confidence: {landmark_conf_disp:.2f})\n" if "box" in landmark_item: box = landmark_item["box"] pt1 = (int(box[0]), int(box[1])); pt2 = (int(box[2]), int(box[3])) color_lm = (255, 0, 255); thickness_lm = 3 # Magenta BGR cv2.rectangle(img_for_drawing_cv2, pt1, pt2, color_lm, thickness_lm) label_lm = f"{landmark_name_disp} ({landmark_conf_disp:.2f})" font_scale_lm = 0.6; font_thickness_lm = 1 (w_text, h_text), baseline = cv2.getTextSize(label_lm, cv2.FONT_HERSHEY_SIMPLEX, font_scale_lm, font_thickness_lm) # Label position logic (simplified from your extensive one for brevity) label_y_pos = pt1[1] - baseline - 3 if label_y_pos < h_text : # If label goes above image, put it below box label_y_pos = pt2[1] + h_text + baseline + 3 label_bg_pt1 = (pt1[0], label_y_pos - h_text - baseline) label_bg_pt2 = (pt1[0] + w_text, label_y_pos + baseline) cv2.rectangle(img_for_drawing_cv2, label_bg_pt1, label_bg_pt2, color_lm, -1) cv2.putText(img_for_drawing_cv2, label_lm, (pt1[0], label_y_pos), cv2.FONT_HERSHEY_SIMPLEX, font_scale_lm, (255,255,255), font_thickness_lm, cv2.LINE_AA) result_image_pil = Image.fromarray(cv2.cvtColor(img_for_drawing_cv2, cv2.COLOR_BGR2RGB)) return result_image_pil, final_result_text, stats_data except Exception as e: error_message = f"Error in ImageProcessor.process_image: {str(e)}" import traceback traceback.print_exc() return pil_image_for_processing if pil_image_for_processing else None, error_message, {} finally: if temp_path and os.path.exists(temp_path): try: os.remove(temp_path) except Exception as e: print(f"Warning: Cannot delete temp file {temp_path}: {str(e)}") def format_result_text(self, stats: Dict) -> str: """ Format detection statistics into readable text with improved spacing Args: stats: Dictionary containing detection statistics Returns: Formatted text summary """ if not stats or "total_objects" not in stats: return "No objects detected." # 減少不必要的空行 lines = [ f"Detected {stats['total_objects']} objects.", f"Average confidence: {stats.get('average_confidence', 0):.2f}", "Objects by class:" ] if "class_statistics" in stats and stats["class_statistics"]: # 按計數排序類別 sorted_classes = sorted( stats["class_statistics"].items(), key=lambda x: x[1]["count"], reverse=True ) for cls_name, cls_stats in sorted_classes: count = cls_stats["count"] conf = cls_stats.get("average_confidence", 0) item_text = "item" if count == 1 else "items" lines.append(f"• {cls_name}: {count} {item_text} (avg conf: {conf:.2f})") else: lines.append("No class information available.") # 添加空間資訊 if "spatial_metrics" in stats and "spatial_distribution" in stats["spatial_metrics"]: lines.append("Object Distribution:") dist = stats["spatial_metrics"]["spatial_distribution"] x_mean = dist.get("x_mean", 0) y_mean = dist.get("y_mean", 0) # 描述物體的大致位置 if x_mean < 0.33: h_pos = "on the left side" elif x_mean < 0.67: h_pos = "in the center" else: h_pos = "on the right side" if y_mean < 0.33: v_pos = "in the upper part" elif y_mean < 0.67: v_pos = "in the middle" else: v_pos = "in the lower part" lines.append(f"• Most objects appear {h_pos} {v_pos} of the image") return "\n".join(lines) def format_json_for_display(self, stats: Dict) -> Dict: """ Format statistics JSON for better display Args: stats: Raw statistics dictionary Returns: Formatted statistics structure for display """ # Create a cleaner copy of the stats for display display_stats = {} # Add summary section display_stats["summary"] = { "total_objects": stats.get("total_objects", 0), "average_confidence": round(stats.get("average_confidence", 0), 3) } # Add class statistics in a more organized way if "class_statistics" in stats and stats["class_statistics"]: # Sort classes by count (descending) sorted_classes = sorted( stats["class_statistics"].items(), key=lambda x: x[1].get("count", 0), reverse=True ) class_stats = {} for cls_name, cls_data in sorted_classes: class_stats[cls_name] = { "count": cls_data.get("count", 0), "average_confidence": round(cls_data.get("average_confidence", 0), 3) } display_stats["detected_objects"] = class_stats # Simplify spatial metrics if "spatial_metrics" in stats: spatial = stats["spatial_metrics"] # Simplify spatial distribution if "spatial_distribution" in spatial: dist = spatial["spatial_distribution"] display_stats["spatial"] = { "distribution": { "x_mean": round(dist.get("x_mean", 0), 3), "y_mean": round(dist.get("y_mean", 0), 3), "x_std": round(dist.get("x_std", 0), 3), "y_std": round(dist.get("y_std", 0), 3) } } # Add simplified size information if "size_distribution" in spatial: size = spatial["size_distribution"] display_stats["spatial"]["size"] = { "mean_area": round(size.get("mean_area", 0), 3), "min_area": round(size.get("min_area", 0), 3), "max_area": round(size.get("max_area", 0), 3) } return display_stats def prepare_visualization_data(self, stats: Dict, available_classes: Dict[int, str]) -> Dict: """ Prepare data for visualization based on detection statistics Args: stats: Detection statistics available_classes: Dictionary of available class IDs and names Returns: Visualization data dictionary """ if not stats or "class_statistics" not in stats or not stats["class_statistics"]: return {"error": "No detection data available"} # Prepare visualization data viz_data = { "total_objects": stats.get("total_objects", 0), "average_confidence": stats.get("average_confidence", 0), "class_data": [] } # Class data for cls_name, cls_stats in stats.get("class_statistics", {}).items(): # Search class ID class_id = -1 for id, name in available_classes.items(): if name == cls_name: class_id = id break cls_data = { "name": cls_name, "class_id": class_id, "count": cls_stats.get("count", 0), "average_confidence": cls_stats.get("average_confidence", 0), "color": self.color_mapper.get_color(class_id if class_id >= 0 else cls_name) } viz_data["class_data"].append(cls_data) # Descending order viz_data["class_data"].sort(key=lambda x: x["count"], reverse=True) return viz_data