VisionScout / scene_analyzer.py
DawnC's picture
Update scene_analyzer.py
ca35826 verified
raw
history blame
112 kB
import os
import numpy as np
from typing import Dict, List, Tuple, Any, Optional
from PIL import Image
from spatial_analyzer import SpatialAnalyzer
from scene_description import SceneDescriptor
from enhance_scene_describer import EnhancedSceneDescriber
from clip_analyzer import CLIPAnalyzer
from landmark_activities import LANDMARK_ACTIVITIES
from clip_zero_shot_classifier import CLIPZeroShotClassifier
from llm_enhancer import LLMEnhancer
from scene_type import SCENE_TYPES
from object_categories import OBJECT_CATEGORIES
from landmark_data import ALL_LANDMARKS
class SceneAnalyzer:
"""
Core class for scene analysis and understanding based on object detection results.
Analyzes detected objects, their relationships, and infers the scene type.
"""
EVERYDAY_SCENE_TYPE_KEYS = [
"general_indoor_space", "generic_street_view",
"desk_area_workspace", "outdoor_gathering_spot",
"kitchen_counter_or_utility_area"
]
def __init__(self, class_names: Dict[int, str] = None, use_llm: bool = True, use_clip: bool = True, enable_landmark=True, llm_model_path: str = None):
"""
Initialize the scene analyzer with optional class name mappings.
Args:
class_names: Dictionary mapping class IDs to class names (optional)
"""
try:
self.class_names = class_names
self.use_clip = use_clip
self.use_landmark_detection = enable_landmark
self.enable_landmark = enable_landmark
# 初始化基本屬性
self.LANDMARK_ACTIVITIES = {}
self.SCENE_TYPES = {}
self.OBJECT_CATEGORIES = {}
# 嘗試加載資料
try:
self.LANDMARK_ACTIVITIES = LANDMARK_ACTIVITIES
print("Loaded LANDMARK_ACTIVITIES successfully")
except Exception as e:
print(f"Warning: Failed to load LANDMARK_ACTIVITIES: {e}")
try:
self.SCENE_TYPES = SCENE_TYPES
print("Loaded SCENE_TYPES successfully")
except Exception as e:
print(f"Warning: Failed to load SCENE_TYPES: {e}")
try:
self.OBJECT_CATEGORIES = OBJECT_CATEGORIES
print("Loaded OBJECT_CATEGORIES successfully")
except Exception as e:
print(f"Warning: Failed to load OBJECT_CATEGORIES: {e}")
# 初始化其他組件
self.spatial_analyzer = None
self.descriptor = None
self.scene_describer = None
try:
self.spatial_analyzer = SpatialAnalyzer(class_names=class_names, object_categories=self.OBJECT_CATEGORIES)
print("Initialized SpatialAnalyzer successfully")
except Exception as e:
print(f"Error initializing SpatialAnalyzer: {e}")
import traceback
traceback.print_exc()
try:
self.descriptor = SceneDescriptor(scene_types=self.SCENE_TYPES, object_categories=self.OBJECT_CATEGORIES)
print("Initialized SceneDescriptor successfully")
except Exception as e:
print(f"Error initializing SceneDescriptor: {e}")
import traceback
traceback.print_exc()
try:
if self.spatial_analyzer:
self.scene_describer = EnhancedSceneDescriber(scene_types=self.SCENE_TYPES, spatial_analyzer_instance=self.spatial_analyzer)
print("Initialized EnhancedSceneDescriber successfully")
else:
print("Warning: Cannot initialize EnhancedSceneDescriber without SpatialAnalyzer")
except Exception as e:
print(f"Error initializing EnhancedSceneDescriber: {e}")
import traceback
traceback.print_exc()
# 初始化 CLIP 分析器
if self.use_clip:
try:
self.clip_analyzer = CLIPAnalyzer()
try:
# 嘗試使用已加載的CLIP模型實例
if hasattr(self.clip_analyzer, 'get_clip_instance'):
model, preprocess, device = self.clip_analyzer.get_clip_instance()
self.landmark_classifier = CLIPZeroShotClassifier(device=device)
print("Initialized landmark classifier with shared CLIP model")
else:
self.landmark_classifier = CLIPZeroShotClassifier()
# 配置地標檢測器
self.landmark_classifier.set_batch_size(8) # 設置合適的批處理大小
self.landmark_classifier.adjust_confidence_threshold("full_image", 0.8) # 整張圖像的閾值要求
self.landmark_classifier.adjust_confidence_threshold("distant", 0.65) # 遠景地標的閾值要求
self.use_landmark_detection = True
print("Landmark detection enabled with optimized settings")
except (ImportError, Exception) as e:
print(f"Warning: Could not initialize landmark classifier: {e}")
self.use_landmark_detection = False
except Exception as e:
print(f"Warning: Could not initialize CLIP analyzer: {e}")
print("Scene analysis will proceed without CLIP. Install CLIP with 'pip install clip' for enhanced scene understanding.")
self.use_clip = False
# 初始化LLM Model
self.use_llm = use_llm
if use_llm:
try:
# from llm_enhancer import LLMEnhancer
self.llm_enhancer = LLMEnhancer(model_path=llm_model_path)
print(f"LLM enhancer initialized successfully.")
except Exception as e:
print(f"Warning: Could not initialize LLM enhancer: {e}")
print("Scene analysis will proceed without LLM. Make sure required packages are installed.")
self.use_llm = False
except Exception as e:
print(f"Critical error during SceneAnalyzer initialization: {e}")
import traceback
traceback.print_exc()
raise
def generate_scene_description(self,
scene_type: str,
detected_objects: List[Dict],
confidence: float,
lighting_info: Optional[Dict] = None,
functional_zones: Optional[Dict] = None,
enable_landmark: bool = True,
scene_scores: Optional[Dict] = None,
spatial_analysis: Optional[Dict] = None,
image_dimensions: Optional[Tuple[int, int]] = None
):
"""
生成場景描述,並將所有必要的上下文傳遞給底層的描述器。
Args:
scene_type: 識別的場景類型
detected_objects: 檢測到的物體列表
confidence: 場景分類置信度
lighting_info: 照明條件信息(可選)
functional_zones: 功能區域信息(可選)
enable_landmark: 是否啟用地標描述(可選)
scene_scores: 場景分數(可選)
spatial_analysis: 空間分析結果(可選)
image_dimensions: 圖像尺寸 (寬, 高)(可選)
Returns:
str: 生成的場景描述
"""
# 轉換 functional_zones 從 Dict 到 List[str],並過濾技術術語
functional_zones_list = []
if functional_zones and isinstance(functional_zones, dict):
# 過濾掉技術術語,只保留有意義的描述
filtered_zones = {k: v for k, v in functional_zones.items()
if not k.endswith('_zone') or k in ['dining_zone', 'seating_zone', 'work_zone']}
functional_zones_list = [v.get('description', k) for k, v in filtered_zones.items()
if isinstance(v, dict) and v.get('description')]
elif functional_zones and isinstance(functional_zones, list):
# 過濾列表中的技術術語
functional_zones_list = [zone for zone in functional_zones
if not zone.endswith('_zone') or 'area' in zone]
# 生成詳細的物體統計信息
object_statistics = {}
for obj in detected_objects:
class_name = obj.get("class_name", "unknown")
if class_name not in object_statistics:
object_statistics[class_name] = {
"count": 0,
"avg_confidence": 0.0,
"max_confidence": 0.0,
"instances": []
}
stats = object_statistics[class_name]
stats["count"] += 1
stats["instances"].append(obj)
stats["max_confidence"] = max(stats["max_confidence"], obj.get("confidence", 0.0))
# 計算平均信心度
for class_name, stats in object_statistics.items():
if stats["count"] > 0:
total_conf = sum(inst.get("confidence", 0.0) for inst in stats["instances"])
stats["avg_confidence"] = total_conf / stats["count"]
return self.scene_describer.generate_description(
scene_type=scene_type,
detected_objects=detected_objects,
confidence=confidence,
lighting_info=lighting_info,
functional_zones=functional_zones_list,
enable_landmark=enable_landmark,
scene_scores=scene_scores,
spatial_analysis=spatial_analysis,
image_dimensions=image_dimensions,
object_statistics=object_statistics
)
def _define_image_regions(self):
"""Define regions of the image for spatial analysis (3x3 grid)"""
self.regions = {
"top_left": (0, 0, 1/3, 1/3),
"top_center": (1/3, 0, 2/3, 1/3),
"top_right": (2/3, 0, 1, 1/3),
"middle_left": (0, 1/3, 1/3, 2/3),
"middle_center": (1/3, 1/3, 2/3, 2/3),
"middle_right": (2/3, 1/3, 1, 2/3),
"bottom_left": (0, 2/3, 1/3, 1),
"bottom_center": (1/3, 2/3, 2/3, 1),
"bottom_right": (2/3, 2/3, 1, 1)
}
def _get_alternative_scene_type(self, landmark_scene_type, detected_objects, scene_scores):
"""
為地標場景類型選擇適合的替代類型
Args:
landmark_scene_type: 原始地標場景類型
detected_objects: 檢測到的物體列表
scene_scores: 所有場景類型的分數
Returns:
str: 適合的替代場景類型
"""
# 1. 嘗試從現有場景分數中找出第二高的非地標場景
landmark_types = {"tourist_landmark", "natural_landmark", "historical_monument"}
alternative_scores = {k: v for k, v in scene_scores.items() if k not in landmark_types and v > 0.2}
if alternative_scores:
# 返回分數最高的非地標場景類型
return max(alternative_scores.items(), key=lambda x: x[1])[0]
# 2. 基於物體組合推斷場景類型
object_counts = {}
for obj in detected_objects:
class_name = obj.get("class_name", "")
if class_name not in object_counts:
object_counts[class_name] = 0
object_counts[class_name] += 1
# 根據物體組合決定場景類型
if "car" in object_counts or "truck" in object_counts or "bus" in object_counts:
# 有車輛,可能是街道或交叉路口
if "traffic light" in object_counts or "stop sign" in object_counts:
return "intersection"
else:
return "city_street"
if "building" in object_counts and object_counts.get("person", 0) > 0:
# 有建築物和人,可能是商業區
return "commercial_district"
if object_counts.get("person", 0) > 3:
# 多個行人,可能是行人區
return "pedestrian_area"
if "bench" in object_counts or "potted plant" in object_counts:
# 有長椅或盆栽,可能是公園區域
return "park_area"
# 3. 根據原始地標場景類型選擇合適的替代場景
if landmark_scene_type == "natural_landmark":
return "outdoor_natural_area"
elif landmark_scene_type == "historical_monument":
return "urban_architecture"
# 默認回退到城市街道
return "city_street"
def analyze(self, detection_result: Any, lighting_info: Optional[Dict] = None, class_confidence_threshold: float = 0.25, scene_confidence_threshold: float = 0.6, enable_landmark=True, places365_info: Optional[Dict] = None) -> Dict:
"""
Analyze detection results to determine scene type and provide understanding.
Args:
detection_result: Detection result from YOLOv8 or similar.
lighting_info: Optional lighting condition analysis results.
class_confidence_threshold: Minimum confidence to consider an object.
scene_confidence_threshold: Minimum confidence to determine a scene.
enable_landmark: Whether to enable landmark detection and recognition for this run.
Returns:
Dictionary with scene analysis results.
"""
current_run_enable_landmark = enable_landmark
print(f"DIAGNOSTIC (SceneAnalyzer.analyze): Called with current_run_enable_landmark={current_run_enable_landmark}")
print(f"DEBUG: SceneAnalyzer received lighting_info type: {type(lighting_info)}")
print(f"DEBUG: SceneAnalyzer lighting_info source: {lighting_info.get('source', 'unknown') if isinstance(lighting_info, dict) else 'not_dict'}")
# Log Places365 information if available
if places365_info:
print(f"DIAGNOSTIC: Places365 info received - scene: {places365_info.get('scene_label', 'unknown')}, "
f"mapped: {places365_info.get('mapped_scene_type', 'unknown')}, "
f"confidence: {places365_info.get('confidence', 0.0):.3f}")
# Sync enable_landmark status with child components for this analysis run
# Assuming these components exist and have an 'enable_landmark' attribute
for component_name in ['scene_describer', 'clip_analyzer', 'landmark_classifier']:
if hasattr(self, component_name):
component = getattr(self, component_name)
if component and hasattr(component, 'enable_landmark'):
component.enable_landmark = current_run_enable_landmark
self.enable_landmark = current_run_enable_landmark # Instance's general state for this run
if hasattr(self, 'use_landmark_detection'):
self.use_landmark_detection = current_run_enable_landmark
original_image_pil = None
image_dims_val = None # Will be (width, height)
if detection_result is not None and hasattr(detection_result, 'orig_img') and detection_result.orig_img is not None:
if isinstance(detection_result.orig_img, np.ndarray):
try:
img_array = detection_result.orig_img
if img_array.ndim == 3 and img_array.shape[2] == 4: # RGBA
img_array = img_array[:, :, :3] # Convert to RGB
if img_array.ndim == 2 : # Grayscale
original_image_pil = Image.fromarray(img_array).convert("RGB")
else: # Assuming RGB or BGR (PIL handles BGR->RGB on fromarray if mode not specified, but explicit is better if source is cv2 BGR)
original_image_pil = Image.fromarray(img_array)
if original_image_pil.mode == 'BGR': # Explicitly convert BGR from OpenCV to RGB for PIL
original_image_pil = original_image_pil.convert('RGB')
image_dims_val = (original_image_pil.width, original_image_pil.height)
except Exception as e:
print(f"Warning: Error converting NumPy orig_img to PIL: {e}")
elif hasattr(detection_result.orig_img, 'size') and callable(getattr(detection_result.orig_img, 'convert', None)):
original_image_pil = detection_result.orig_img.copy().convert("RGB") # Ensure RGB
image_dims_val = original_image_pil.size
else:
print(f"Warning: detection_result.orig_img (type: {type(detection_result.orig_img)}) is not a recognized NumPy array or PIL Image.")
else:
print("Warning: detection_result.orig_img not available. Image-based analysis will be limited.")
# Handling cases with no YOLO detections (or no boxes attribute)
no_yolo_detections = (detection_result is None or
not hasattr(detection_result, 'boxes') or
not hasattr(detection_result.boxes, 'xyxy') or
len(detection_result.boxes.xyxy) == 0)
if no_yolo_detections:
tried_landmark_detection = False
landmark_detection_result = None
if original_image_pil and self.use_clip and current_run_enable_landmark:
if not hasattr(self, 'landmark_classifier') and hasattr(self, 'clip_analyzer'):
try:
if hasattr(self.clip_analyzer, 'get_clip_instance'):
model, preprocess, device = self.clip_analyzer.get_clip_instance()
self.landmark_classifier = CLIPZeroShotClassifier(device=device)
print("Initialized landmark classifier with shared CLIP model")
else:
self.landmark_classifier = CLIPZeroShotClassifier()
print("Created landmark classifier on demand for no YOLO detection path")
except Exception as e:
print(f"Warning: Could not initialize landmark classifier: {e}")
# 地標搜索
if hasattr(self, 'landmark_classifier'):
try:
tried_landmark_detection = True
print("Attempting landmark detection with no YOLO boxes")
landmark_results_no_yolo = self.landmark_classifier.intelligent_landmark_search(
original_image_pil, yolo_boxes=None, base_threshold=0.2 # 略微降低閾值,提高靈敏度
)
# 確保在無地標場景時返回有效結果
if landmark_results_no_yolo is None:
landmark_results_no_yolo = {"is_landmark_scene": False, "detected_landmarks": []}
if landmark_results_no_yolo and landmark_results_no_yolo.get("is_landmark_scene", False):
primary_landmark_no_yolo = landmark_results_no_yolo.get("primary_landmark")
# 放寬閾值條件,以便捕獲更多潛在地標
if primary_landmark_no_yolo and primary_landmark_no_yolo.get("confidence", 0) > 0.25: # 降低閾值
landmark_detection_result = True
detected_objects_from_landmarks_list = []
w_img_no_yolo, h_img_no_yolo = image_dims_val if image_dims_val else (1,1)
for lm_info_item in landmark_results_no_yolo.get("detected_landmarks", []):
if lm_info_item.get("confidence", 0) > 0.25: # 降低閾值與上面保持一致
# 安全獲取 box 值,避免索引錯誤
box = lm_info_item.get("box", [0, 0, w_img_no_yolo, h_img_no_yolo])
# 確保 box 包含至少 4 個元素
if len(box) < 4:
box = [0, 0, w_img_no_yolo, h_img_no_yolo]
# 計算中心點和標準化坐標
center_x, center_y = (box[0] + box[2]) / 2, (box[1] + box[3]) / 2
norm_cx = center_x / w_img_no_yolo if w_img_no_yolo > 0 else 0.5
norm_cy = center_y / h_img_no_yolo if h_img_no_yolo > 0 else 0.5
# 決定地標類型
landmark_type = "architectural" # 預設類型
landmark_id = lm_info_item.get("landmark_id", "")
if hasattr(self.landmark_classifier, '_determine_landmark_type') and landmark_id:
try:
landmark_type = self.landmark_classifier._determine_landmark_type(landmark_id)
except Exception as e:
print(f"Error determining landmark type: {e}")
else:
# 使用簡單的基於 ID 的啟發式方法推斷類型
landmark_id_lower = landmark_id.lower() if isinstance(landmark_id, str) else ""
if "natural" in landmark_id_lower or any(term in landmark_id_lower for term in ["mountain", "waterfall", "canyon", "lake"]):
landmark_type = "natural"
elif "monument" in landmark_id_lower or "memorial" in landmark_id_lower or "historical" in landmark_id_lower:
landmark_type = "monument"
# 決定區域位置
region = "center" # 預設值
if hasattr(self, 'spatial_analyzer') and hasattr(self.spatial_analyzer, '_determine_region'):
try:
region = self.spatial_analyzer._determine_region(norm_cx, norm_cy)
except Exception as e:
print(f"Error determining region: {e}")
# 創建地標物體
landmark_obj = {
"class_id": lm_info_item.get("landmark_id", f"LM_{lm_info_item.get('landmark_name','unk')}")[:15],
"class_name": lm_info_item.get("landmark_name", "Unknown Landmark"),
"confidence": lm_info_item.get("confidence", 0.0),
"box": box,
"center": (center_x, center_y),
"normalized_center": (norm_cx, norm_cy),
"size": (box[2] - box[0], box[3] - box[1]),
"normalized_size": (
(box[2] - box[0])/(w_img_no_yolo if w_img_no_yolo>0 else 1),
(box[3] - box[1])/(h_img_no_yolo if h_img_no_yolo>0 else 1)
),
"area": (box[2] - box[0]) * (box[3] - box[1]),
"normalized_area": (
(box[2] - box[0]) * (box[3] - box[1])
) / ((w_img_no_yolo*h_img_no_yolo) if w_img_no_yolo*h_img_no_yolo >0 else 1),
"is_landmark": True,
"landmark_id": landmark_id,
"location": lm_info_item.get("location", "Unknown Location"),
"region": region,
"year_built": lm_info_item.get("year_built", ""),
"architectural_style": lm_info_item.get("architectural_style", ""),
"significance": lm_info_item.get("significance", ""),
"landmark_type": landmark_type
}
detected_objects_from_landmarks_list.append(landmark_obj)
if detected_objects_from_landmarks_list:
# 設定場景類型
best_scene_val_no_yolo = "tourist_landmark" # 預設
if primary_landmark_no_yolo:
try:
lm_type_no_yolo = primary_landmark_no_yolo.get("landmark_type", "architectural")
if lm_type_no_yolo and "natural" in lm_type_no_yolo.lower():
best_scene_val_no_yolo = "natural_landmark"
elif lm_type_no_yolo and ("historical" in lm_type_no_yolo.lower() or "monument" in lm_type_no_yolo.lower()):
best_scene_val_no_yolo = "historical_monument"
except Exception as e:
print(f"Error determining scene type from landmark type: {e}")
# 確保場景類型有效
if not hasattr(self, 'SCENE_TYPES') or best_scene_val_no_yolo not in self.SCENE_TYPES:
best_scene_val_no_yolo = "tourist_landmark" # 預設場景類型
# 設定置信度
scene_confidence_no_yolo = primary_landmark_no_yolo.get("confidence", 0.0) if primary_landmark_no_yolo else 0.0
# 分析空間區域
region_analysis_for_lm_desc = {}
if hasattr(self, 'spatial_analyzer') and hasattr(self.spatial_analyzer, '_analyze_regions'):
try:
region_analysis_for_lm_desc = self.spatial_analyzer._analyze_regions(detected_objects_from_landmarks_list)
except Exception as e:
print(f"Error analyzing regions: {e}")
# 獲取功能區
f_zones_no_yolo = {}
if hasattr(self, 'spatial_analyzer') and hasattr(self.spatial_analyzer, '_identify_landmark_zones'):
try:
f_zones_no_yolo = self.spatial_analyzer._identify_landmark_zones(detected_objects_from_landmarks_list)
except Exception as e:
print(f"Error identifying landmark zones: {e}")
# 生成場景描述
scene_desc_no_yolo = f"A {best_scene_val_no_yolo} scene." # 預設描述
if hasattr(self, 'scene_describer') and hasattr(self.scene_describer, 'generate_description'):
try:
scene_desc_no_yolo = self.scene_describer.generate_description(
scene_type=best_scene_val_no_yolo,
detected_objects=detected_objects_from_landmarks_list,
confidence=scene_confidence_no_yolo,
lighting_info=lighting_info,
functional_zones=list(f_zones_no_yolo.keys()) if f_zones_no_yolo else [],
enable_landmark=True,
scene_scores={best_scene_val_no_yolo: scene_confidence_no_yolo},
spatial_analysis=region_analysis_for_lm_desc,
image_dimensions=image_dims_val
)
except Exception as e:
print(f"Error generating scene description: {e}")
# 使用 LLM 增強描述
enhanced_desc_no_yolo = scene_desc_no_yolo
if self.use_llm and hasattr(self, 'llm_enhancer') and hasattr(self.llm_enhancer, 'enhance_description'):
try:
# 準備用於 LLM 增強器的數據
prominent_objects_detail = ""
if hasattr(self, 'scene_describer') and hasattr(self.scene_describer, '_format_object_list_for_description'):
try:
prominent_objects_detail = self.scene_describer._format_object_list_for_description(
detected_objects_from_landmarks_list[:min(1, len(detected_objects_from_landmarks_list))]
)
except Exception as e:
print(f"Error formatting object list: {e}")
scene_data_llm_no_yolo = {
"original_description": scene_desc_no_yolo,
"scene_type": best_scene_val_no_yolo,
"scene_name": self.SCENE_TYPES.get(best_scene_val_no_yolo, {}).get("name", "Landmark")
if hasattr(self, 'SCENE_TYPES') else "Landmark",
"detected_objects": detected_objects_from_landmarks_list,
"object_list": "landmark",
"confidence": scene_confidence_no_yolo,
"lighting_info": lighting_info,
"functional_zones": f_zones_no_yolo,
"clip_analysis": landmark_results_no_yolo.get("clip_analysis_on_full_image", {}),
"enable_landmark": True,
"image_width": w_img_no_yolo,
"image_height": h_img_no_yolo,
"prominent_objects_detail": prominent_objects_detail
}
enhanced_desc_no_yolo = self.llm_enhancer.enhance_description(scene_data_llm_no_yolo)
except Exception as e:
print(f"Error enhancing description with LLM: {e}")
import traceback
traceback.print_exc()
# 計算可能的活動,優先使用地標特定活動
possible_activities = ["Sightseeing"]
# 檢查是否有主要地標活動從 CLIP 分析結果中獲取
primary_landmark_activities = landmark_results_no_yolo.get("primary_landmark_activities", [])
if primary_landmark_activities:
print(f"Using {len(primary_landmark_activities)} landmark-specific activities")
possible_activities = primary_landmark_activities
else:
# 從檢測到的地標中提取特定活動
landmark_specific_activities = []
for lm_info_item in landmark_results_no_yolo.get("detected_landmarks", []):
lm_id = lm_info_item.get("landmark_id")
if lm_id and hasattr(self, 'LANDMARK_ACTIVITIES') and lm_id in self.LANDMARK_ACTIVITIES:
landmark_specific_activities.extend(self.LANDMARK_ACTIVITIES[lm_id])
if landmark_specific_activities:
possible_activities = list(set(landmark_specific_activities)) # 去重
print(f"Extracted {len(possible_activities)} activities from landmark data")
else:
# 回退到通用活動推斷
if hasattr(self, 'descriptor') and hasattr(self.descriptor, '_infer_possible_activities'):
try:
possible_activities = self.descriptor._infer_possible_activities(
best_scene_val_no_yolo,
detected_objects_from_landmarks_list,
enable_landmark=True,
scene_scores={best_scene_val_no_yolo: scene_confidence_no_yolo}
)
except Exception as e:
print(f"Error inferring possible activities: {e}")
# 準備最終結果
return {
"scene_type": best_scene_val_no_yolo,
"scene_name": self.SCENE_TYPES.get(best_scene_val_no_yolo, {}).get("name", "Landmark")
if hasattr(self, 'SCENE_TYPES') else "Landmark",
"confidence": round(float(scene_confidence_no_yolo), 4),
"description": scene_desc_no_yolo,
"enhanced_description": enhanced_desc_no_yolo,
"objects_present": detected_objects_from_landmarks_list,
"object_count": len(detected_objects_from_landmarks_list),
"regions": region_analysis_for_lm_desc,
"possible_activities": possible_activities,
"functional_zones": f_zones_no_yolo,
"detected_landmarks": [lm for lm in detected_objects_from_landmarks_list if lm.get("is_landmark", False)],
"primary_landmark": primary_landmark_no_yolo,
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
except Exception as e:
print(f"Error in landmark-only detection path (analyze method): {e}")
import traceback
traceback.print_exc()
# 如果地標檢測失敗或未嘗試,使用 CLIP 進行一般場景分析
if not landmark_detection_result and self.use_clip and original_image_pil:
try:
clip_analysis_val_no_yolo = None
if hasattr(self, 'clip_analyzer') and hasattr(self.clip_analyzer, 'analyze_image'):
try:
clip_analysis_val_no_yolo = self.clip_analyzer.analyze_image(
original_image_pil,
enable_landmark=current_run_enable_landmark
)
except Exception as e:
print(f"Error in CLIP analysis: {e}")
scene_type_llm_no_yolo = "llm_inferred_no_yolo"
confidence_llm_no_yolo = 0.0
if clip_analysis_val_no_yolo and isinstance(clip_analysis_val_no_yolo, dict):
top_scene = clip_analysis_val_no_yolo.get("top_scene")
if top_scene and isinstance(top_scene, tuple) and len(top_scene) >= 2:
confidence_llm_no_yolo = top_scene[1]
if isinstance(top_scene[0], str):
scene_type_llm_no_yolo = top_scene[0]
desc_llm_no_yolo = "Primary object detection did not yield results. This description is based on overall image context."
w_llm_no_yolo, h_llm_no_yolo = image_dims_val if image_dims_val else (1, 1)
enhanced_desc_llm_no_yolo = desc_llm_no_yolo
if self.use_llm and hasattr(self, 'llm_enhancer'):
try:
# 確保數據正確格式化
clip_analysis_safe = {}
if isinstance(clip_analysis_val_no_yolo, dict):
clip_analysis_safe = clip_analysis_val_no_yolo
scene_data_llm_no_yolo_enhance = {
"original_description": desc_llm_no_yolo,
"scene_type": scene_type_llm_no_yolo,
"scene_name": "Contextually Inferred (No Detections)",
"detected_objects": [],
"object_list": "general ambiance",
"confidence": confidence_llm_no_yolo,
"lighting_info": lighting_info or {"time_of_day": "unknown", "confidence": 0.0},
"clip_analysis": clip_analysis_safe,
"enable_landmark": current_run_enable_landmark,
"image_width": w_llm_no_yolo,
"image_height": h_llm_no_yolo,
"prominent_objects_detail": "the overall visual context"
}
if hasattr(self.llm_enhancer, 'enhance_description'):
try:
enhanced_desc_llm_no_yolo = self.llm_enhancer.enhance_description(scene_data_llm_no_yolo_enhance)
except Exception as e:
print(f"Error in enhance_description: {e}")
if (not enhanced_desc_llm_no_yolo or len(enhanced_desc_llm_no_yolo.strip()) < 20) and hasattr(self.llm_enhancer, 'handle_no_detection'):
try:
enhanced_desc_llm_no_yolo = self.llm_enhancer.handle_no_detection(clip_analysis_safe)
except Exception as e:
print(f"Error in handle_no_detection: {e}")
except Exception as e:
print(f"Error preparing data for LLM enhancement: {e}")
import traceback
traceback.print_exc()
# 安全類型轉換
try:
confidence_float = float(confidence_llm_no_yolo)
except (ValueError, TypeError):
confidence_float = 0.0
# 確保增強描述不為空
if not enhanced_desc_llm_no_yolo or not isinstance(enhanced_desc_llm_no_yolo, str):
enhanced_desc_llm_no_yolo = desc_llm_no_yolo
# 返回結果
return {
"scene_type": scene_type_llm_no_yolo,
"confidence": round(confidence_float, 4),
"description": desc_llm_no_yolo,
"enhanced_description": enhanced_desc_llm_no_yolo,
"objects_present": [],
"object_count": 0,
"regions": {},
"possible_activities": [],
"safety_concerns": [],
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
except Exception as e:
print(f"Error in CLIP no-detection fallback (analyze method): {e}")
import traceback
traceback.print_exc()
# Check if Places365 provides useful scene information even without YOLO detections
fallback_scene_type = "unknown"
fallback_confidence = 0.0
fallback_description = "No objects were detected in the image, and contextual analysis could not be performed or failed."
if places365_info and places365_info.get('confidence', 0) > 0.3:
fallback_scene_type = places365_info.get('mapped_scene_type', 'unknown')
fallback_confidence = places365_info.get('confidence', 0.0)
fallback_description = f"Scene appears to be {places365_info.get('scene_label', 'an unidentified location')} based on overall visual context."
return {
"scene_type": fallback_scene_type,
"confidence": fallback_confidence,
"description": fallback_description,
"enhanced_description": "The image analysis system could not detect any recognizable objects or landmarks in this image.",
"objects_present": [],
"object_count": 0,
"regions": {},
"possible_activities": [],
"safety_concerns": [],
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
if self.use_llm and self.use_clip and original_image_pil:
try:
clip_analysis_val_no_yolo = self.clip_analyzer.analyze_image(original_image_pil, enable_landmark=current_run_enable_landmark)
scene_type_llm_no_yolo = "llm_inferred_no_yolo"
confidence_llm_no_yolo = clip_analysis_val_no_yolo.get("top_scene", ("unknown", 0.0))[1] if isinstance(clip_analysis_val_no_yolo, dict) else 0.0
desc_llm_no_yolo = "Primary object detection did not yield results. This description is based on overall image context."
w_llm_no_yolo, h_llm_no_yolo = image_dims_val if image_dims_val else (1,1)
scene_data_llm_no_yolo_enhance = {
"original_description": desc_llm_no_yolo, "scene_type": scene_type_llm_no_yolo,
"scene_name": "Contextually Inferred (No Detections)", "detected_objects": [], "object_list": "general ambiance",
"confidence": confidence_llm_no_yolo, "lighting_info": lighting_info, "clip_analysis": clip_analysis_val_no_yolo,
"enable_landmark": current_run_enable_landmark, "image_width": w_llm_no_yolo, "image_height": h_llm_no_yolo,
"prominent_objects_detail": "the overall visual context"
}
enhanced_desc_llm_no_yolo = self.llm_enhancer.enhance_description(scene_data_llm_no_yolo_enhance) if hasattr(self, 'llm_enhancer') else desc_llm_no_yolo
if hasattr(self, 'llm_enhancer') and hasattr(self.llm_enhancer, 'handle_no_detection') and (not enhanced_desc_llm_no_yolo or len(enhanced_desc_llm_no_yolo.strip()) < 20):
enhanced_desc_llm_no_yolo = self.llm_enhancer.handle_no_detection(clip_analysis_val_no_yolo)
return {
"scene_type": scene_type_llm_no_yolo, "confidence": round(float(confidence_llm_no_yolo),4),
"description": desc_llm_no_yolo, "enhanced_description": enhanced_desc_llm_no_yolo,
"objects_present": [], "object_count": 0, "regions": {}, "possible_activities": [],
"safety_concerns": [], "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
except Exception as e:
print(f"Error in LLM/CLIP no-detection fallback (analyze method): {e}")
return {
"scene_type": "unknown", "confidence": 0.0,
"description": "No objects were detected in the image, and contextual analysis could not be performed or failed.",
"objects_present": [], "object_count": 0, "regions": {}, "possible_activities": [],
"safety_concerns": [], "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
# Main processing flow if YOLO detections are present
if self.class_names is None and hasattr(detection_result, 'names'):
self.class_names = detection_result.names
if hasattr(self.spatial_analyzer, 'class_names'):
self.spatial_analyzer.class_names = self.class_names
detected_objects_main = self.spatial_analyzer._extract_detected_objects(
detection_result,
confidence_threshold=class_confidence_threshold
)
if not detected_objects_main:
return {
"scene_type": "unknown", "confidence": 0.0,
"description": "No objects detected with sufficient confidence by the primary vision system.",
"objects_present": [], "object_count": 0, "regions": {}, "possible_activities": [],
"safety_concerns": [], "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
# Spatial analysis done once on YOLO objects
region_analysis_val = self.spatial_analyzer._analyze_regions(detected_objects_main)
final_functional_zones = {}
final_activities = []
final_landmark_info = {}
tentative_best_scene = "unknown"
tentative_scene_confidence = 0.0
# Landmark Processing and Integration
landmark_objects_identified_clip = []
landmark_specific_activities = [] # NEW
if self.use_clip and current_run_enable_landmark and hasattr(self, 'process_unknown_objects') and hasattr(self, 'landmark_classifier'):
detected_objects_main_after_lm, landmark_objects_identified_clip = self.process_unknown_objects(
detection_result,
detected_objects_main
)
detected_objects_main = detected_objects_main_after_lm # Update main list
if landmark_objects_identified_clip:
primary_landmark_clip = max(landmark_objects_identified_clip, key=lambda x: x.get("confidence", 0.0), default=None)
if primary_landmark_clip and primary_landmark_clip.get("confidence", 0.0) > 0.35:
lm_type_raw = "architectural" # Default
if hasattr(self.landmark_classifier, '_determine_landmark_type') and primary_landmark_clip.get("landmark_id"):
lm_type_raw = self.landmark_classifier._determine_landmark_type(primary_landmark_clip.get("landmark_id"))
else:
lm_type_raw = primary_landmark_clip.get("landmark_type", "architectural")
if lm_type_raw == "natural": tentative_best_scene = "natural_landmark"
elif lm_type_raw == "monument": tentative_best_scene = "historical_monument"
else: tentative_best_scene = "tourist_landmark"
tentative_scene_confidence = primary_landmark_clip.get("confidence", 0.0)
final_landmark_info = {
"detected_landmarks": landmark_objects_identified_clip,
"primary_landmark": primary_landmark_clip,
"detailed_landmarks": landmark_objects_identified_clip
}
# 專門儲存地標特定活動的列表
landmark_specific_activities = []
# 優先收集來自識別地標的特定活動
for lm_obj in landmark_objects_identified_clip:
lm_id = lm_obj.get("landmark_id")
if lm_id and lm_id in self.LANDMARK_ACTIVITIES:
landmark_specific_activities.extend(self.LANDMARK_ACTIVITIES[lm_id])
# 將特定地標活動加入最終活動列表
if landmark_specific_activities:
final_activities.extend(landmark_specific_activities)
print(f"Added {len(landmark_specific_activities)} landmark-specific activities for {', '.join([lm.get('landmark_name', 'unknown') for lm in landmark_objects_identified_clip if lm.get('is_landmark', False)])}")
if hasattr(self.spatial_analyzer, '_identify_landmark_zones'):
final_functional_zones.update(self.spatial_analyzer._identify_landmark_zones(landmark_objects_identified_clip))
if not current_run_enable_landmark:
detected_objects_main = [obj for obj in detected_objects_main if not obj.get("is_landmark", False)]
final_landmark_info = {}
# --- Compute YOLO-based scene scores ---
# MODIFIED: Pass region_analysis_val as spatial_analysis_results
yolo_scene_scores_val = self._compute_scene_scores(detected_objects_main,
spatial_analysis_results=region_analysis_val)
# --- CLIP Analysis for general scene scores ---
clip_scene_scores_val = {}
clip_analysis_results = None # To store the full dict from clip_analyzer
if self.use_clip and original_image_pil is not None:
try:
clip_analysis_results = self.clip_analyzer.analyze_image(
original_image_pil,
enable_landmark=current_run_enable_landmark,
exclude_categories=["landmark", "tourist", "monument", "tower", "attraction", "scenic", "historical", "famous"] if not current_run_enable_landmark else None
)
if isinstance(clip_analysis_results, dict): # Ensure it's a dict before get
clip_scene_scores_val = clip_analysis_results.get("scene_scores", {})
# Filter again if landmarks are disabled
if not current_run_enable_landmark:
clip_scene_scores_val = {k: v for k, v in clip_scene_scores_val.items() if not any(kw in k.lower() for kw in ["landmark", "monument", "tourist"])}
if "cultural_analysis" in clip_analysis_results: del clip_analysis_results["cultural_analysis"]
if "top_scene" in clip_analysis_results and any(term in clip_analysis_results.get("top_scene",["unknown",0.0])[0].lower() for term in ["landmark", "monument", "tourist"]):
non_lm_cs = sorted([item for item in clip_scene_scores_val.items() if item[1] > 0], key=lambda x:x[1], reverse=True)
clip_analysis_results["top_scene"] = non_lm_cs[0] if non_lm_cs else ("unknown", 0.0)
# (Keep your asian_commercial_street special handling here if needed)
if not lighting_info and "lighting_condition" in clip_analysis_results: # If main lighting_info is still None
lt, lc = clip_analysis_results.get("lighting_condition", ("unknown", 0.0))
lighting_info = {"time_of_day": lt, "confidence": lc, "source": "CLIP_fallback"}
except Exception as e:
print(f"Error in main CLIP analysis for YOLO path (analyze method): {e}")
# Calculate stats for _fuse_scene_scores (based on non-landmark YOLO objects)
yolo_only_objects_for_fuse_stats = [obj for obj in detected_objects_main if not obj.get("is_landmark")]
num_yolo_detections_for_fuse = len(yolo_only_objects_for_fuse_stats)
avg_yolo_confidence_for_fuse = sum(obj.get('confidence', 0.0) for obj in yolo_only_objects_for_fuse_stats) / num_yolo_detections_for_fuse if num_yolo_detections_for_fuse > 0 else 0.0
print(f"DEBUG: About to call _fuse_scene_scores with lighting_info: {lighting_info}")
print(f"DEBUG: Places365_info being passed to fuse: {places365_info}")
scene_scores_fused = self._fuse_scene_scores(
yolo_scene_scores_val, clip_scene_scores_val,
num_yolo_detections=num_yolo_detections_for_fuse,
avg_yolo_confidence=avg_yolo_confidence_for_fuse,
lighting_info=lighting_info,
places365_info=places365_info
)
# Respect tentative scene from strong landmark detection during fusion adjustment
if tentative_best_scene != "unknown" and "landmark" in tentative_best_scene.lower() and tentative_scene_confidence > 0.5:
scene_scores_fused[tentative_best_scene] = max(scene_scores_fused.get(tentative_best_scene, 0.0), tentative_scene_confidence * 0.95)
# Final determination of scene type
final_best_scene, final_scene_confidence = self._determine_scene_type(scene_scores_fused)
if not current_run_enable_landmark and final_best_scene in ["tourist_landmark", "natural_landmark", "historical_monument"]:
if hasattr(self, '_get_alternative_scene_type'):
alt_scene_type = self._get_alternative_scene_type(final_best_scene, detected_objects_main, scene_scores_fused)
final_best_scene = alt_scene_type
final_scene_confidence = scene_scores_fused.get(alt_scene_type, 0.6)
else:
final_best_scene = "generic_street_view"; final_scene_confidence = min(final_scene_confidence, 0.65)
# Generate final descriptive content (Activities, Safety, Zones)
# 如果有特定地標活動,限制通用活動的數量
generic_activities = []
if hasattr(self.descriptor, '_infer_possible_activities'):
generic_activities = self.descriptor._infer_possible_activities(
final_best_scene, detected_objects_main,
enable_landmark=current_run_enable_landmark, scene_scores=scene_scores_fused
)
# 優先處理策略:使用特定地標活動,不足時才從通用活動補充
if landmark_specific_activities:
# 如果有特定活動,優先保留,去除與特定活動重複的通用活動
unique_generic_activities = [act for act in generic_activities if act not in landmark_specific_activities]
# 如果特定活動少於3個,從通用活動中補充
if len(landmark_specific_activities) < 3:
# 補充通用活動但總數不超過7個
supplement_count = min(3 - len(landmark_specific_activities), len(unique_generic_activities))
if supplement_count > 0:
final_activities.extend(unique_generic_activities[:supplement_count])
else:
# 若無特定活動,則使用所有通用活動
final_activities.extend(generic_activities)
# 去重並排序,但確保特定地標活動保持在前面
final_activities_set = set(final_activities)
final_activities = []
# 先加入特定地標活動(按原順序)
for activity in landmark_specific_activities:
if activity in final_activities_set:
final_activities.append(activity)
final_activities_set.remove(activity)
# 再加入通用活動(按字母排序)
final_activities.extend(sorted(list(final_activities_set)))
final_safety_concerns = self.descriptor._identify_safety_concerns(detected_objects_main, final_best_scene) if hasattr(self.descriptor, '_identify_safety_concerns') else []
if hasattr(self.spatial_analyzer, '_identify_functional_zones'): # Update functional_zones
general_zones = self.spatial_analyzer._identify_functional_zones(detected_objects_main, final_best_scene)
for gz_key, gz_val in general_zones.items():
if gz_key not in final_functional_zones: final_functional_zones[gz_key] = gz_val
# Filter again if landmarks disabled for this run
if not current_run_enable_landmark:
final_functional_zones = {k: v for k, v in final_functional_zones.items() if not any(kw in k.lower() for kw in ["landmark", "monument", "viewing", "tourist"])}
current_activities_temp = [act for act in final_activities if not any(kw in act.lower() for kw in ["sightsee", "photograph", "tour", "histor", "landmark", "monument", "cultur"])]
final_activities = current_activities_temp
if not final_activities and hasattr(self.descriptor, '_infer_possible_activities'):
final_activities = self.descriptor._infer_possible_activities("generic_street_view", detected_objects_main, enable_landmark=False)
# 創建淨化的光線資訊,避免不合理的時間描述
lighting_info_clean = None
if lighting_info:
lighting_info_clean = {
"is_indoor": lighting_info.get("is_indoor"),
"confidence": lighting_info.get("confidence", 0.0),
"time_of_day": lighting_info.get("time_of_day", "unknown") # 加入這行
}
# 如果 Places365 提供高信心度判斷,就用它的結果
if places365_info and places365_info.get('confidence', 0) >= 0.8:
lighting_info_clean["is_indoor"] = places365_info.get('is_indoor')
lighting_info_clean["confidence"] = places365_info.get('confidence')
base_scene_description = self.generate_scene_description(
scene_type=final_best_scene,
detected_objects=detected_objects_main,
confidence=final_scene_confidence,
lighting_info=lighting_info_clean,
functional_zones=final_functional_zones,
enable_landmark=current_run_enable_landmark,
scene_scores=scene_scores_fused,
spatial_analysis=region_analysis_val,
image_dimensions=image_dims_val
)
if not current_run_enable_landmark and hasattr(self, '_remove_landmark_references'):
base_scene_description = self._remove_landmark_references(base_scene_description)
# --- LLM Enhancement ---
enhanced_final_description = base_scene_description
llm_verification_output = None
if self.use_llm and hasattr(self, 'llm_enhancer'):
try:
obj_list_for_llm = ", ".join(sorted(list(set(
obj["class_name"] for obj in detected_objects_main
if obj.get("confidence", 0) > 0.4 and not obj.get("is_landmark")
))))
if not obj_list_for_llm and current_run_enable_landmark and final_landmark_info.get("primary_landmark"):
obj_list_for_llm = final_landmark_info["primary_landmark"].get("class_name", "a prominent feature")
elif not obj_list_for_llm: obj_list_for_llm = "various visual elements"
# 生成物體統計信息
object_statistics = {}
for obj in detected_objects_main:
class_name = obj.get("class_name", "unknown")
if class_name not in object_statistics:
object_statistics[class_name] = {
"count": 0,
"avg_confidence": 0.0,
"max_confidence": 0.0,
"instances": []
}
stats = object_statistics[class_name]
stats["count"] += 1
stats["instances"].append(obj)
stats["max_confidence"] = max(stats["max_confidence"], obj.get("confidence", 0.0))
# 計算平均信心度
for class_name, stats in object_statistics.items():
if stats["count"] > 0:
total_conf = sum(inst.get("confidence", 0.0) for inst in stats["instances"])
stats["avg_confidence"] = total_conf / stats["count"]
llm_scene_data = {
"original_description": base_scene_description, "scene_type": final_best_scene,
"scene_name": self.SCENE_TYPES.get(final_best_scene, {}).get("name", "Unknown Scene"),
"detected_objects": detected_objects_main, "object_list": obj_list_for_llm,
"object_statistics": object_statistics, # 新增統計信息
"confidence": final_scene_confidence, "lighting_info": lighting_info,
"functional_zones": final_functional_zones, "activities": final_activities,
"safety_concerns": final_safety_concerns,
"clip_analysis": clip_analysis_results if isinstance(clip_analysis_results, dict) else None,
"enable_landmark": current_run_enable_landmark,
"image_width": image_dims_val[0] if image_dims_val else None,
"image_height": image_dims_val[1] if image_dims_val else None,
"prominent_objects_detail": self.scene_describer._format_object_list_for_description(
self.scene_describer._get_prominent_objects(detected_objects_main, min_prominence_score=0.1, max_categories_to_return=3, max_total_objects=7)
) if hasattr(self.scene_describer, '_get_prominent_objects') and hasattr(self.scene_describer, '_format_object_list_for_description') else ""
}
if current_run_enable_landmark and final_landmark_info.get("primary_landmark"):
llm_scene_data["primary_landmark_info"] = final_landmark_info["primary_landmark"]
if self.use_clip and clip_analysis_results and isinstance(clip_analysis_results, dict) and "top_scene" in clip_analysis_results:
clip_top_name = clip_analysis_results.get("top_scene",["unknown",0.0])[0]
clip_top_conf = clip_analysis_results.get("top_scene",["unknown",0.0])[1]
if clip_top_name != final_best_scene and clip_top_conf > 0.4 and final_scene_confidence > 0.4 and hasattr(self.llm_enhancer, 'verify_detection'):
llm_verification_output = self.llm_enhancer.verify_detection(
detected_objects_main, clip_analysis_results, final_best_scene,
self.SCENE_TYPES.get(final_best_scene, {}).get("name", "Unknown"), final_scene_confidence
)
if llm_verification_output : llm_scene_data["verification_result"] = llm_verification_output.get("verification_text", "")
enhanced_final_description = self.llm_enhancer.enhance_description(llm_scene_data)
if not current_run_enable_landmark and hasattr(self, '_remove_landmark_references'):
enhanced_final_description = self._remove_landmark_references(enhanced_final_description)
except Exception as e:
print(f"Error in LLM Enhancement in main flow (analyze method): {e}")
# Construct final output dictionary
output_result = {
"scene_type": final_best_scene if final_scene_confidence >= scene_confidence_threshold else "unknown",
"scene_name": self.SCENE_TYPES.get(final_best_scene, {}).get("name", "Unknown Scene") if final_scene_confidence >= scene_confidence_threshold else "Unknown Scene",
"confidence": round(float(final_scene_confidence), 4),
"description": base_scene_description,
"enhanced_description": enhanced_final_description,
"objects_present": [{"class_id": obj.get("class_id", -1), "class_name": obj.get("class_name", "unknown"), "confidence": round(float(obj.get("confidence",0.0)), 4)} for obj in detected_objects_main],
"object_count": len(detected_objects_main),
"regions": region_analysis_val,
"possible_activities": final_activities,
"safety_concerns": final_safety_concerns,
"functional_zones": final_functional_zones,
"alternative_scenes": self.descriptor._get_alternative_scenes(scene_scores_fused, scene_confidence_threshold, top_k=2) if hasattr(self.descriptor, '_get_alternative_scenes') else [],
"lighting_conditions": lighting_info if lighting_info else {"time_of_day": "unknown", "confidence": 0.0, "source": "default"}
}
if current_run_enable_landmark and final_landmark_info and final_landmark_info.get("detected_landmarks"):
output_result.update(final_landmark_info)
if final_best_scene in ["tourist_landmark", "natural_landmark", "historical_monument"]:
output_result["scene_source"] = "landmark_detection"
elif not current_run_enable_landmark:
for key_rm in ["detected_landmarks", "primary_landmark", "detailed_landmarks", "scene_source"]:
if key_rm in output_result: del output_result[key_rm]
if llm_verification_output:
output_result["llm_verification"] = llm_verification_output.get("verification_text")
if llm_verification_output.get("has_errors", False):
output_result["detection_warnings"] = "LLM detected potential issues with object recognition."
if clip_analysis_results and isinstance(clip_analysis_results, dict) and "error" not in clip_analysis_results:
top_scene_clip = clip_analysis_results.get("top_scene", ("unknown", 0.0))
output_result["clip_analysis"] = {
"top_scene": (top_scene_clip[0], round(float(top_scene_clip[1]), 4)),
"cultural_analysis": clip_analysis_results.get("cultural_analysis", {}) if current_run_enable_landmark else {}
}
return output_result
def _get_object_spatial_cohesion_score(self, objects_for_scene: List[Dict], spatial_analysis_results: Optional[Dict]) -> float:
"""
(This is a NEW helper function)
Calculates a score based on how spatially cohesive the key objects for a scene are.
A higher score means objects are more clustered in fewer regions.
This is a heuristic and can be refined.
Args:
objects_for_scene: List of detected objects (dictionaries with at least 'class_id')
relevant to the current scene type being evaluated.
spatial_analysis_results: Output from SpatialAnalyzer._analyze_regions.
Expected format: {'objects_by_region': {'region_name': [{'class_id': id, ...}, ...]}}
Returns:
float: A cohesion score, typically a small bonus (e.g., 0.0 to 0.1).
"""
if not objects_for_scene or not spatial_analysis_results or \
"objects_by_region" not in spatial_analysis_results or \
not spatial_analysis_results["objects_by_region"]:
return 0.0
# Get the set of class_ids for the key objects defining the current scene type
key_object_class_ids = {obj.get('class_id') for obj in objects_for_scene if obj.get('class_id') is not None}
if not key_object_class_ids:
return 0.0
# Find in which regions these key objects appear
regions_containing_key_objects = set()
# Count how many of the *instances* of key objects are found
# This helps differentiate a scene with 1 chair in 1 region vs 5 chairs spread over 5 regions
total_key_object_instances_found = 0
for region_name, objects_in_region_list in spatial_analysis_results["objects_by_region"].items():
region_has_key_object = False
for obj_in_region in objects_in_region_list:
if obj_in_region.get('class_id') in key_object_class_ids:
region_has_key_object = True
total_key_object_instances_found += 1 # Count each instance
if region_has_key_object:
regions_containing_key_objects.add(region_name)
num_distinct_key_objects_in_scene = len(key_object_class_ids) # Number of *types* of key objects
num_instances_of_key_objects_passed = len(objects_for_scene) # Number of *instances* passed for this scene
if not regions_containing_key_objects or num_instances_of_key_objects_passed == 0:
return 0.0
# A simple heuristic:
if len(regions_containing_key_objects) == 1 and total_key_object_instances_found >= num_instances_of_key_objects_passed * 0.75:
return 0.10 # Strongest cohesion: most/all key object instances in a single region
elif len(regions_containing_key_objects) <= 2 and total_key_object_instances_found >= num_instances_of_key_objects_passed * 0.60:
return 0.05 # Moderate cohesion: most/all key object instances in up to two regions
elif len(regions_containing_key_objects) <= 3 and total_key_object_instances_found >= num_instances_of_key_objects_passed * 0.50:
return 0.02 # Weaker cohesion
return 0.0
def _compute_scene_scores(self, detected_objects: List[Dict], spatial_analysis_results: Optional[Dict] = None) -> Dict[str, float]:
"""
Compute confidence scores for each scene type based on detected objects.
Enhanced to better score everyday scenes and consider object richness and spatial cohesion.
Args:
detected_objects: List of detected objects with their details (class_id, confidence, region, etc.).
spatial_analysis_results: Optional output from SpatialAnalyzer, specifically 'objects_by_region',
which is used by _get_object_spatial_cohesion_score.
Returns:
Dictionary mapping scene types to confidence scores.
"""
scene_scores = {}
if not detected_objects:
for scene_type_key in self.SCENE_TYPES:
scene_scores[scene_type_key] = 0.0
return scene_scores
# Prepare data from detected_objects
detected_class_ids_all = [obj["class_id"] for obj in detected_objects]
detected_classes_set_all = set(detected_class_ids_all)
class_counts_all = {}
for obj in detected_objects:
class_id = obj["class_id"]
class_counts_all[class_id] = class_counts_all.get(class_id, 0) + 1
# Evaluate each scene type defined in self.SCENE_TYPES
for scene_type, scene_def in self.SCENE_TYPES.items():
required_obj_ids_defined = set(scene_def.get("required_objects", []))
optional_obj_ids_defined = set(scene_def.get("optional_objects", []))
min_required_matches_needed = scene_def.get("minimum_required", 0)
# Determine which actual detected objects are relevant for this scene_type
# These lists will store the actual detected object dicts, not just class_ids
actual_required_objects_found_list = []
for req_id in required_obj_ids_defined:
if req_id in detected_classes_set_all:
# Find first instance of this required object to add to list (for cohesion check later)
for dobj in detected_objects:
if dobj['class_id'] == req_id:
actual_required_objects_found_list.append(dobj)
break
num_required_matches_found = len(actual_required_objects_found_list)
actual_optional_objects_found_list = []
for opt_id in optional_obj_ids_defined:
if opt_id in detected_classes_set_all:
for dobj in detected_objects:
if dobj['class_id'] == opt_id:
actual_optional_objects_found_list.append(dobj)
break
num_optional_matches_found = len(actual_optional_objects_found_list)
# --- Initial Score Calculation Weights ---
# Base score: 55% from required, 25% from optional, 10% richness, 10% cohesion (max)
required_weight = 0.55
optional_weight = 0.25
richness_bonus_max = 0.10
cohesion_bonus_max = 0.10 # Max bonus from _get_object_spatial_cohesion_score is 0.1
current_scene_score = 0.0
objects_to_check_for_cohesion = [] # For spatial cohesion scoring
# --- Check minimum_required condition & Calculate base score ---
if num_required_matches_found >= min_required_matches_needed:
if len(required_obj_ids_defined) > 0:
required_ratio = num_required_matches_found / len(required_obj_ids_defined)
else: # No required objects defined, but min_required_matches_needed could be 0
required_ratio = 1.0 if min_required_matches_needed == 0 else 0.0
current_scene_score = required_ratio * required_weight
objects_to_check_for_cohesion.extend(actual_required_objects_found_list)
# Add score from optional objects
if len(optional_obj_ids_defined) > 0:
optional_ratio = num_optional_matches_found / len(optional_obj_ids_defined)
current_scene_score += optional_ratio * optional_weight
objects_to_check_for_cohesion.extend(actual_optional_objects_found_list)
# Flexible handling for "everyday scenes" if strict minimum_required (based on 'required_objects') isn't met
elif scene_type in self.EVERYDAY_SCENE_TYPE_KEYS:
# If an everyday scene has many optional items, it might still be a weak candidate
# Check if a decent proportion of its 'optional_objects' are present
if len(optional_obj_ids_defined) > 0 and \
(num_optional_matches_found / len(optional_obj_ids_defined)) >= 0.25: # e.g., at least 25% of typical optional items
# Base score more on optional fulfillment for these types
current_scene_score = (num_optional_matches_found / len(optional_obj_ids_defined)) * (required_weight + optional_weight * 0.5) # Give some base
objects_to_check_for_cohesion.extend(actual_optional_objects_found_list)
else:
scene_scores[scene_type] = 0.0
continue # Skip this scene type
else: # For non-everyday scenes, if minimum_required is not met, score is 0
scene_scores[scene_type] = 0.0
continue
# --- Bonus for object richness/variety ---
# Considers unique object *classes* found that are relevant to the scene definition
relevant_defined_class_ids = required_obj_ids_defined.union(optional_obj_ids_defined)
unique_relevant_detected_classes = relevant_defined_class_ids.intersection(detected_classes_set_all)
object_richness_score = 0.0
if len(relevant_defined_class_ids) > 0:
richness_ratio = len(unique_relevant_detected_classes) / len(relevant_defined_class_ids)
object_richness_score = min(richness_bonus_max, richness_ratio * 0.15) # Max 10% bonus from richness
current_scene_score += object_richness_score
# --- Bonus for spatial cohesion (if spatial_analysis_results are provided) ---
spatial_cohesion_bonus = 0.0
if spatial_analysis_results and objects_to_check_for_cohesion:
# Deduplicate objects_to_check_for_cohesion based on actual object instances (not just class_id)
# This can be done by converting list of dicts to list of tuples of items for hashing
# However, assuming _get_object_spatial_cohesion_score handles instances correctly.
# If objects_to_check_for_cohesion might have duplicate dict references for the SAME object,
# then a more robust deduplication on actual object references would be needed if not already handled.
# For now, assume it's a list of unique object *instances* found relevant to the scene.
spatial_cohesion_bonus = self._get_object_spatial_cohesion_score(
objects_to_check_for_cohesion, # Pass the list of actual detected object dicts
spatial_analysis_results
)
current_scene_score += spatial_cohesion_bonus # Max 0.1 from this bonus
# --- Bonus for multiple instances of key objects (original logic refined) ---
multiple_instance_bonus = 0.0
# For multiple instance bonus, focus on objects central to the scene's definition
key_objects_for_multi_instance_check = required_obj_ids_defined
if scene_type in self.EVERYDAY_SCENE_TYPE_KEYS and len(optional_obj_ids_defined) > 0:
# For everyday scenes, some optionals can also be key if they appear multiple times
# e.g., multiple chairs in a "general_indoor_space"
key_objects_for_multi_instance_check = key_objects_for_multi_instance_check.union(
set(list(optional_obj_ids_defined)[:max(1, len(optional_obj_ids_defined)//2)]) # consider first half of optionals
)
for class_id_check in key_objects_for_multi_instance_check:
if class_id_check in detected_classes_set_all and class_counts_all.get(class_id_check, 0) > 1:
multiple_instance_bonus += 0.025 # Slightly smaller bonus per type
current_scene_score += min(0.075, multiple_instance_bonus) # Max 7.5% bonus
# Apply scene-specific priority defined in SCENE_TYPES
if "priority" in scene_def:
current_scene_score *= scene_def["priority"]
scene_scores[scene_type] = min(1.0, max(0.0, current_scene_score))
# If landmark detection is disabled via the instance attribute self.enable_landmark,
# ensure scores for landmark-specific scene types are zeroed out.
if hasattr(self, 'enable_landmark') and not self.enable_landmark:
landmark_scene_types = ["tourist_landmark", "natural_landmark", "historical_monument"]
for lm_scene_type in landmark_scene_types:
if lm_scene_type in scene_scores:
scene_scores[lm_scene_type] = 0.0
return scene_scores
def _determine_scene_type(self, scene_scores: Dict[str, float]) -> Tuple[str, float]:
"""
Determine the most likely scene type based on scores.
Args:
scene_scores: Dictionary mapping scene types to confidence scores
Returns:
Tuple of (best_scene_type, confidence)
"""
if not scene_scores:
return "unknown", 0.0
best_scene = max(scene_scores, key=scene_scores.get)
best_score = scene_scores[best_scene]
return best_scene, float(best_score)
def _fuse_scene_scores(self,
yolo_scene_scores: Dict[str, float],
clip_scene_scores: Dict[str, float],
num_yolo_detections: int = 0,
avg_yolo_confidence: float = 0.0,
lighting_info: Optional[Dict] = None,
places365_info: Optional[Dict] = None
) -> Dict[str, float]:
"""
Fuse scene scores from YOLO-based object detection, CLIP-based analysis, and Places365 scene classification.
Adjusts weights based on scene type, richness of YOLO detections, lighting information, and Places365 confidence.
Args:
yolo_scene_scores: Scene scores based on YOLO object detection.
clip_scene_scores: Scene scores based on CLIP analysis.
num_yolo_detections: Total number of non-landmark objects detected by YOLO with sufficient confidence.
avg_yolo_confidence: Average confidence of non-landmark objects detected by YOLO.
lighting_info: Optional lighting condition analysis results,
expected to contain 'is_indoor' (bool) and 'confidence' (float).
places365_info: Optional Places365 scene classification results,
expected to contain 'mapped_scene_type', 'confidence', and 'is_indoor'.
Returns:
Dict: Fused scene scores incorporating all three analysis sources.
"""
# Handle cases where one of the score dictionaries might be empty or all scores are effectively zero
# Extract and process Places365 scene scores
places365_scene_scores_map = {} # 修改變數名稱以避免與傳入的字典衝突
if places365_info and places365_info.get('confidence', 0) > 0.1:
mapped_scene_type = places365_info.get('mapped_scene_type', 'unknown')
places365_confidence = places365_info.get('confidence', 0.0)
if mapped_scene_type in self.SCENE_TYPES.keys():
places365_scene_scores_map[mapped_scene_type] = places365_confidence # 使用新的字典
print(f"Places365 contributing: {mapped_scene_type} with confidence {places365_confidence:.3f}")
yolo_has_meaningful_scores = bool(yolo_scene_scores and any(s > 1e-5 for s in yolo_scene_scores.values())) # 確保是布林值
clip_has_meaningful_scores = bool(clip_scene_scores and any(s > 1e-5 for s in clip_scene_scores.values())) # 確保是布林值
places365_has_meaningful_scores = bool(places365_scene_scores_map and any(s > 1e-5 for s in places365_scene_scores_map.values()))
meaningful_sources_count = sum([
yolo_has_meaningful_scores,
clip_has_meaningful_scores,
places365_has_meaningful_scores
])
if meaningful_sources_count == 0:
return {st: 0.0 for st in self.SCENE_TYPES.keys()}
elif meaningful_sources_count == 1:
if yolo_has_meaningful_scores:
return {st: yolo_scene_scores.get(st, 0.0) for st in self.SCENE_TYPES.keys()}
elif clip_has_meaningful_scores:
return {st: clip_scene_scores.get(st, 0.0) for st in self.SCENE_TYPES.keys()}
elif places365_has_meaningful_scores:
return {st: places365_scene_scores_map.get(st, 0.0) for st in self.SCENE_TYPES.keys()}
fused_scores = {}
all_relevant_scene_types = set(self.SCENE_TYPES.keys())
all_possible_scene_types = all_relevant_scene_types.union(
set(yolo_scene_scores.keys()),
set(clip_scene_scores.keys()),
set(places365_scene_scores_map.keys())
)
# Base weights - adjusted to accommodate three sources
default_yolo_weight = 0.5
default_clip_weight = 0.3
default_places365_weight = 0.2
is_lighting_indoor = None
lighting_analysis_confidence = 0.0
if lighting_info and isinstance(lighting_info, dict):
is_lighting_indoor = lighting_info.get("is_indoor")
lighting_analysis_confidence = lighting_info.get("confidence", 0.0)
for scene_type in all_possible_scene_types:
yolo_score = yolo_scene_scores.get(scene_type, 0.0)
clip_score = clip_scene_scores.get(scene_type, 0.0)
places365_score = places365_scene_scores_map.get(scene_type, 0.0)
current_yolo_weight = default_yolo_weight
current_clip_weight = default_clip_weight
current_places365_weight = default_places365_weight
scene_definition = self.SCENE_TYPES.get(scene_type, {})
# Weight adjustment based on scene_type nature and YOLO richness
if scene_type in self.EVERYDAY_SCENE_TYPE_KEYS:
# Places365 excels at everyday scene classification
if num_yolo_detections >= 5 and avg_yolo_confidence >= 0.45: # Rich YOLO for everyday
current_yolo_weight = 0.60
current_clip_weight = 0.15
current_places365_weight = 0.25
elif num_yolo_detections >= 3: # Moderate YOLO for everyday
current_yolo_weight = 0.50
current_clip_weight = 0.20
current_places365_weight = 0.30
else: # Sparse YOLO for everyday, rely more on Places365
current_yolo_weight = 0.35
current_clip_weight = 0.25
current_places365_weight = 0.40
# For scenes where CLIP's global understanding or specific training is often more valuable
elif any(keyword in scene_type.lower() for keyword in ["asian", "cultural", "aerial", "landmark", "monument", "tourist", "natural_landmark", "historical_monument"]):
current_yolo_weight = 0.25
current_clip_weight = 0.65
current_places365_weight = 0.10 # Lower weight for landmark scenes
# For specific indoor common scenes (non-landmark), object detection is key but Places365 provides strong scene context
elif any(keyword in scene_type.lower() for keyword in
["room", "kitchen", "office", "bedroom", "desk_area", "indoor_space",
"professional_kitchen", "cafe", "library", "gym", "retail_store",
"supermarket", "classroom", "conference_room", "medical_facility",
"educational_setting", "dining_area"]):
current_yolo_weight = 0.55
current_clip_weight = 0.20
current_places365_weight = 0.25
# For specific outdoor common scenes (non-landmark) where objects are still important
elif any(keyword in scene_type.lower() for keyword in
["parking_lot", "park_area", "beach", "harbor", "playground", "sports_field", "bus_stop", "train_station", "airport"]):
current_yolo_weight = 0.50
current_clip_weight = 0.25
current_places365_weight = 0.25
# If landmark detection is globally disabled for this run
if hasattr(self, 'enable_landmark') and not self.enable_landmark:
if any(keyword in scene_type.lower() for keyword in ["landmark", "monument", "tourist"]):
yolo_score = 0.0 # Should already be 0 from _compute_scene_scores
clip_score *= 0.05 # Heavily penalize
places365_score *= 0.8 if scene_type not in self.EVERYDAY_SCENE_TYPE_KEYS else 1.0 # Slight penalty for landmark scenes
elif scene_type not in self.EVERYDAY_SCENE_TYPE_KEYS and \
not any(keyword in scene_type.lower() for keyword in ["asian", "cultural", "aerial"]):
# Redistribute weights away from CLIP towards YOLO and Places365
weight_boost = 0.05
current_yolo_weight = min(0.9, current_yolo_weight + weight_boost)
current_places365_weight = min(0.9, current_places365_weight + weight_boost)
current_clip_weight = max(0.1, current_clip_weight - weight_boost * 2)
# Boost Places365 weight if it has high confidence for this specific scene type
if places365_score > 0.0 and places365_info: # 這裡的 places365_score 已經是從 map 中獲取
places365_original_confidence = places365_info.get('confidence', 0.0) # 獲取原始的 Places365 信心度
if places365_original_confidence > 0.7:
boost_factor = min(0.2, (places365_original_confidence - 0.7) * 0.4)
current_places365_weight += boost_factor
total_other_weight = current_yolo_weight + current_clip_weight
if total_other_weight > 0:
reduction_factor = boost_factor / total_other_weight
current_yolo_weight *= (1 - reduction_factor)
current_clip_weight *= (1 - reduction_factor)
total_weight = current_yolo_weight + current_clip_weight + current_places365_weight
if total_weight > 0: # 避免除以零
current_yolo_weight /= total_weight
current_clip_weight /= total_weight
current_places365_weight /= total_weight
else:
current_yolo_weight = 1/3
current_clip_weight = 1/3
current_places365_weight = 1/3
fused_score = (yolo_score * current_yolo_weight) + (clip_score * current_clip_weight) + (places365_score * current_places365_weight)
places365_is_indoor = None
places365_confidence_for_indoor = 0.0
effective_is_indoor = is_lighting_indoor
effective_confidence = lighting_analysis_confidence
if places365_info and isinstance(places365_info, dict):
places365_is_indoor = places365_info.get('is_indoor')
places365_confidence_for_indoor = places365_info.get('confidence', 0.0)
# Places365 overrides lighting analysis when confidence is high
if places365_confidence_for_indoor >= 0.8 and places365_is_indoor is not None:
effective_is_indoor = places365_is_indoor
effective_confidence = places365_confidence_for_indoor
# 只在特定場景類型首次處理時輸出調試資訊
if scene_type == "intersection" or (scene_type in ["urban_intersection", "street_view"] and scene_type == sorted(all_possible_scene_types)[0]):
print(f"DEBUG: Using Places365 indoor/outdoor decision: {places365_is_indoor} (confidence: {places365_confidence_for_indoor:.3f}) over lighting analysis")
if effective_is_indoor is not None and effective_confidence >= 0.65:
# Determine if the scene_type is inherently indoor or outdoor based on its definition
is_defined_as_indoor = "indoor" in scene_definition.get("description", "").lower() or \
any(kw in scene_type.lower() for kw in ["room", "kitchen", "office", "indoor", "library", "cafe", "gym"])
is_defined_as_outdoor = "outdoor" in scene_definition.get("description", "").lower() or \
any(kw in scene_type.lower() for kw in ["street", "park", "aerial", "beach", "harbor", "intersection", "crosswalk"])
lighting_adjustment_strength = 0.20 # Max adjustment factor (e.g., 20%)
# Scale adjustment by how confident the analysis is above the threshold
adjustment_scale = (effective_confidence - 0.65) / (1.0 - 0.65) # Scale from 0 to 1
adjustment = lighting_adjustment_strength * adjustment_scale
adjustment = min(lighting_adjustment_strength, max(0, adjustment)) # Clamp adjustment
if effective_is_indoor and is_defined_as_outdoor:
fused_score *= (1.0 - adjustment)
elif not effective_is_indoor and is_defined_as_indoor:
fused_score *= (1.0 - adjustment)
elif effective_is_indoor and is_defined_as_indoor:
fused_score = min(1.0, fused_score * (1.0 + adjustment * 0.5))
elif not effective_is_indoor and is_defined_as_outdoor:
fused_score = min(1.0, fused_score * (1.0 + adjustment * 0.5))
fused_scores[scene_type] = min(1.0, max(0.0, fused_score))
return fused_scores
def process_unknown_objects(self, detection_result, detected_objects):
"""
對YOLO未能識別或信心度低的物體進行地標檢測
Args:
detection_result: YOLO檢測結果
detected_objects: 已識別的物體列表
Returns:
tuple: (更新後的物體列表, 地標物體列表)
"""
if not getattr(self, 'enable_landmark', True) or not self.use_clip or not hasattr(self, 'use_landmark_detection') or not self.use_landmark_detection:
# 未啟用地標識別時,確保返回的物體列表中不包含任何地標物體
cleaned_objects = [obj for obj in detected_objects if not obj.get("is_landmark", False)]
return cleaned_objects, []
try:
# 獲取原始圖像
original_image = None
if detection_result is not None and hasattr(detection_result, 'orig_img'):
original_image = detection_result.orig_img
# 檢查原始圖像是否存在
if original_image is None:
print("Warning: Original image not available for landmark detection")
return detected_objects, []
# 確保原始圖像為PIL格式或可轉換為PIL格式
if not isinstance(original_image, Image.Image):
if isinstance(original_image, np.ndarray):
try:
if original_image.ndim == 3 and original_image.shape[2] == 4: # RGBA
original_image = original_image[:, :, :3] # 轉換為RGB
if original_image.ndim == 2: # 灰度圖
original_image = Image.fromarray(original_image).convert("RGB")
else: # 假設為RGB或BGR
original_image = Image.fromarray(original_image)
if hasattr(original_image, 'mode') and original_image.mode == 'BGR': # 從OpenCV明確將BGR轉換為RGB
original_image = original_image.convert('RGB')
except Exception as e:
print(f"Warning: Error converting image for landmark detection: {e}")
return detected_objects, []
else:
print(f"Warning: Cannot process image of type {type(original_image)}")
return detected_objects, []
# 獲取圖像維度
if isinstance(original_image, np.ndarray):
h, w = original_image.shape[:2]
elif isinstance(original_image, Image.Image):
w, h = original_image.size
else:
print(f"Warning: Unable to determine image dimensions for type {type(original_image)}")
return detected_objects, []
# 收集可能含有地標的區域
candidate_boxes = []
low_conf_boxes = []
# 即使沒有YOLO檢測到的物體,也嘗試進行更詳細的地標分析
if len(detected_objects) == 0:
# 創建一個包含整個圖像的框
full_image_box = [0, 0, w, h]
low_conf_boxes.append(full_image_box)
candidate_boxes.append((full_image_box, "full_image"))
# 加入網格分析以增加檢測成功率
grid_size = 2 # 2x2網格
for i in range(grid_size):
for j in range(grid_size):
# 創建網格框
grid_box = [
j * w / grid_size,
i * h / grid_size,
(j + 1) * w / grid_size,
(i + 1) * h / grid_size
]
low_conf_boxes.append(grid_box)
candidate_boxes.append((grid_box, "grid"))
# 創建更大的中心框(覆蓋中心70%區域)
center_box = [
w * 0.15, h * 0.15,
w * 0.85, h * 0.85
]
low_conf_boxes.append(center_box)
candidate_boxes.append((center_box, "center"))
print("No YOLO detections, attempting detailed landmark analysis with multiple regions")
else:
try:
# 獲取原始YOLO檢測結果中的低置信度物體
if hasattr(detection_result, 'boxes') and hasattr(detection_result.boxes, 'xyxy') and hasattr(detection_result.boxes, 'conf') and hasattr(detection_result.boxes, 'cls'):
all_boxes = detection_result.boxes.xyxy.cpu().numpy() if hasattr(detection_result.boxes.xyxy, 'cpu') else detection_result.boxes.xyxy
all_confs = detection_result.boxes.conf.cpu().numpy() if hasattr(detection_result.boxes.conf, 'cpu') else detection_result.boxes.conf
all_cls = detection_result.boxes.cls.cpu().numpy() if hasattr(detection_result.boxes.cls, 'cpu') else detection_result.boxes.cls
# 收集低置信度區域和可能含有地標的區域(如建築物)
for i, (box, conf, cls) in enumerate(zip(all_boxes, all_confs, all_cls)):
is_low_conf = conf < 0.4 and conf > 0.1
# 根據物體類別 ID 識別建築物 - 使用通用分類
common_building_classes = [11, 12, 13, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65] # 常見建築類別 ID
is_building = int(cls) in common_building_classes
# 計算相對面積 - 大物體
is_large_object = (box[2] - box[0]) * (box[3] - box[1]) > (0.1 * w * h)
if is_low_conf or is_building:
# 確保 box 是一個有效的數組或列表
if isinstance(box, (list, tuple, np.ndarray)) and len(box) >= 4:
low_conf_boxes.append(box)
if is_large_object:
candidate_boxes.append((box, "building" if is_building else "low_conf"))
except Exception as e:
print(f"Error processing YOLO detections: {e}")
import traceback
traceback.print_exc()
if not hasattr(self, 'landmark_classifier'):
if hasattr(self, 'clip_analyzer') and hasattr(self.clip_analyzer, 'get_clip_instance'):
try:
print("Initializing landmark classifier for process_unknown_objects")
model, preprocess, device = self.clip_analyzer.get_clip_instance()
self.landmark_classifier = CLIPZeroShotClassifier(device=device)
except Exception as e:
print(f"Error initializing landmark classifier: {e}")
return detected_objects, []
else:
print("Warning: landmark_classifier not available and cannot be initialized")
return detected_objects, []
# 使用智能地標搜索
landmark_results = None
try:
# 確保有有效的框
if not low_conf_boxes:
# 如果沒有低置信度框,添加全圖
low_conf_boxes.append([0, 0, w, h])
landmark_results = self.landmark_classifier.intelligent_landmark_search(
original_image,
yolo_boxes=low_conf_boxes,
base_threshold=0.25
)
except Exception as e:
print(f"Error in intelligent_landmark_search: {e}")
import traceback
traceback.print_exc()
return detected_objects, []
# 處理識別結果
landmark_objects = []
# 如果有效的地標結果
if landmark_results and landmark_results.get("is_landmark_scene", False):
for landmark_info in landmark_results.get("detected_landmarks", []):
try:
# 使用 landmark_classifier 的閾值判斷
base_threshold = 0.25 # 基礎閾值
# 獲取地標類型並設定閾值
landmark_type = "architectural" # 預設類型
type_threshold = 0.5 # 預設閾值
# 優先使用 landmark_classifier
if hasattr(self, 'landmark_classifier') and hasattr(self.landmark_classifier, '_determine_landmark_type'):
landmark_type = self.landmark_classifier._determine_landmark_type(landmark_info.get("landmark_id", ""))
type_threshold = getattr(self.landmark_classifier, 'landmark_type_thresholds', {}).get(landmark_type, 0.5)
# 否則使用本地方法
elif hasattr(self, '_determine_landmark_type'):
landmark_type = self._determine_landmark_type(landmark_info.get("landmark_id", ""))
# 依據地標類型調整閾值
if landmark_type == "skyscraper":
type_threshold = 0.4
elif landmark_type == "natural":
type_threshold = 0.6
# 或者直接從地標 ID 推斷
else:
landmark_id = landmark_info.get("landmark_id", "").lower()
if any(term in landmark_id for term in ["mountain", "canyon", "waterfall", "lake", "river", "natural"]):
landmark_type = "natural"
type_threshold = 0.6
elif any(term in landmark_id for term in ["skyscraper", "building", "tower", "tall"]):
landmark_type = "skyscraper"
type_threshold = 0.4
elif any(term in landmark_id for term in ["monument", "memorial", "statue", "historical"]):
landmark_type = "monument"
type_threshold = 0.5
effective_threshold = base_threshold * (type_threshold / 0.5)
# 如果置信度足夠高
if landmark_info.get("confidence", 0) > effective_threshold:
# 獲取邊界框
if "box" in landmark_info:
box = landmark_info["box"]
else:
# 如果沒有邊界框,使用整個圖像的90%區域
margin_x, margin_y = w * 0.05, h * 0.05
box = [margin_x, margin_y, w - margin_x, h - margin_y]
# 計算中心點和其他必要信息
center_x = (box[0] + box[2]) / 2
center_y = (box[1] + box[3]) / 2
norm_center_x = center_x / w if w > 0 else 0.5
norm_center_y = center_y / h if h > 0 else 0.5
# 獲取區域位置
region = "center" # 預設
if hasattr(self, 'spatial_analyzer') and hasattr(self.spatial_analyzer, '_determine_region'):
try:
region = self.spatial_analyzer._determine_region(norm_center_x, norm_center_y)
except Exception as e:
print(f"Error determining region: {e}")
# 創建地標物體
landmark_obj = {
"class_id": landmark_info.get("landmark_id", "")[:15] if isinstance(landmark_info.get("landmark_id", ""), str) else "-100", # 截斷過長的 ID
"class_name": landmark_info.get("landmark_name", "Unknown Landmark"),
"confidence": landmark_info.get("confidence", 0.0),
"box": box,
"center": (center_x, center_y),
"normalized_center": (norm_center_x, norm_center_y),
"size": (box[2] - box[0], box[3] - box[1]),
"normalized_size": (
(box[2] - box[0]) / w if w > 0 else 0,
(box[3] - box[1]) / h if h > 0 else 0
),
"area": (box[2] - box[0]) * (box[3] - box[1]),
"normalized_area": (
(box[2] - box[0]) * (box[3] - box[1]) / (w * h) if w * h > 0 else 0
),
"region": region,
"is_landmark": True,
"landmark_id": landmark_info.get("landmark_id", ""),
"location": landmark_info.get("location", "Unknown Location")
}
# 添加額外信息
for key in ["year_built", "architectural_style", "significance"]:
if key in landmark_info:
landmark_obj[key] = landmark_info[key]
# 添加地標類型
landmark_obj["landmark_type"] = landmark_type
# 添加到檢測物體列表
detected_objects.append(landmark_obj)
landmark_objects.append(landmark_obj)
print(f"Detected landmark: {landmark_info.get('landmark_name', 'Unknown')} with confidence {landmark_info.get('confidence', 0.0):.2f}")
except Exception as e:
print(f"Error processing landmark: {e}")
continue
return detected_objects, landmark_objects
return detected_objects, []
except Exception as e:
print(f"Error in landmark detection: {e}")
import traceback
traceback.print_exc()
return detected_objects, []
def _remove_landmark_references(self, text):
"""
從文本中移除所有地標引用
Args:
text: 輸入文本
Returns:
str: 清除地標引用後的文本
"""
if not text:
return text
import re
try:
# 動態收集所有地標名稱和位置
landmark_names = []
locations = []
for landmark_id, info in ALL_LANDMARKS.items():
# 收集地標名稱及其別名
landmark_names.append(info["name"])
landmark_names.extend(info.get("aliases", []))
# 收集地理位置
if "location" in info:
location = info["location"]
locations.append(location)
# 處理分離的城市和國家名稱
parts = location.split(",")
if len(parts) >= 1:
locations.append(parts[0].strip())
if len(parts) >= 2:
locations.append(parts[1].strip())
# 使用正則表達式動態替換所有地標名稱
for name in landmark_names:
if name and len(name) > 2: # 避免過短的名稱
text = re.sub(r'\b' + re.escape(name) + r'\b', "tall structure", text, flags=re.IGNORECASE)
# 動態替換所有位置引用
for location in locations:
if location and len(location) > 2:
# 替換常見位置表述模式
text = re.sub(r'in ' + re.escape(location), "in the urban area", text, flags=re.IGNORECASE)
text = re.sub(r'of ' + re.escape(location), "of the urban area", text, flags=re.IGNORECASE)
text = re.sub(r'\b' + re.escape(location) + r'\b', "the urban area", text, flags=re.IGNORECASE)
except ImportError:
# 通用地標描述模式
landmark_patterns = [
# 地標地點模式
(r'an iconic structure in ([A-Z][a-zA-Z\s,]+)', r'an urban structure'),
(r'a famous (monument|tower|landmark) in ([A-Z][a-zA-Z\s,]+)', r'an urban structure'),
(r'(the [A-Z][a-zA-Z\s]+ Tower)', r'the tower'),
(r'(the [A-Z][a-zA-Z\s]+ Building)', r'the building'),
(r'(the CN Tower)', r'the tower'),
(r'([A-Z][a-zA-Z\s]+) Tower', r'tall structure'),
# 地標位置關係模式
(r'(centered|built|located|positioned) around the ([A-Z][a-zA-Z\s]+? (Tower|Monument|Landmark))', r'located in this area'),
# 地標活動模式
(r'(sightseeing|guided tours|cultural tourism) (at|around|near) (this landmark|the [A-Z][a-zA-Z\s]+)', r'\1 in this area'),
# 一般性地標形容模式
(r'this (famous|iconic|historic|well-known) (landmark|monument|tower|structure)', r'this urban structure'),
(r'landmark scene', r'urban scene'),
(r'tourist destination', r'urban area'),
(r'tourist attraction', r'urban area')
]
for pattern, replacement in landmark_patterns:
text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
return text