attempt pano image cropping

app.py

@@ -1,4 +1,6 @@
 import streamlit as st
+import cv2
+from ultralytics import YOLO  # For street detection
 import folium
 from streamlit_folium import st_folium
 import requests
@@ -32,6 +34,117 @@ if not MAPILLARY_ACCESS_TOKEN:
     st.error("Mapillary access token not found. Please configure it in the Space secrets.")
     st.stop()
 
+def detect_and_crop_street(panorama_url, use_yolo=True):
+    """
+    Detect streets in a panoramic image and return a cropped normal-sized image
+    Args:
+        panorama_url: URL of the panoramic image
+        use_yolo: Whether to use YOLOv8 (True) or simple edge detection (False)
+    Returns:
+        cropped_image: PIL Image containing the cropped street view
+    """
+    # Download and convert image to CV2 format
+    response = requests.get(panorama_url)
+    img = Image.open(BytesIO(response.content))
+    cv_img = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+    if use_yolo:
+        # Load YOLOv8 model
+        model = YOLO('yolov8n.pt')
+
+        # Detect objects
+        results = model(cv_img)
+
+        # Look for road/street class (index 0 in COCO dataset)
+        street_boxes = []
+        for result in results:
+            for box, cls in zip(result.boxes.xyxy, result.boxes.cls):
+                if cls == 0:  # road class
+                    street_boxes.append(box.cpu().numpy())
+
+        if street_boxes:
+            # Take the largest street detection
+            largest_box = max(street_boxes, key=lambda box: (box[2]-box[0])*(box[3]-box[1]))
+            x1, y1, x2, y2 = map(int, largest_box)
+
+            # Add some padding
+            padding = 50
+            height, width = cv_img.shape[:2]
+            x1 = max(0, x1 - padding)
+            y1 = max(0, y1 - padding)
+            x2 = min(width, x2 + padding)
+            y2 = min(height, y2 + padding)
+
+            cropped = cv_img[y1:y2, x1:x2]
+        else:
+            # Fallback to edge detection if no streets found
+            cropped = edge_based_crop(cv_img)
+    else:
+        cropped = edge_based_crop(cv_img)
+
+    # Convert back to PIL Image
+    cropped_pil = Image.fromarray(cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB))
+
+    # Resize to standard dimensions while maintaining aspect ratio
+    target_width = 1024
+    aspect_ratio = cropped.shape[1] / cropped.shape[0]
+    target_height = int(target_width / aspect_ratio)
+    cropped_pil = cropped_pil.resize((target_width, target_height), Image.Resampling.LANCZOS)
+
+    return cropped_pil
+
+def edge_based_crop(cv_img):
+    """
+    Use edge detection to find and crop around street areas
+    """
+    # Convert to grayscale
+    gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
+
+    # Apply Gaussian blur
+    blurred = cv2.GaussianBlur(gray, (5, 5), 0)
+
+    # Detect edges
+    edges = cv2.Canny(blurred, 50, 150)
+
+    # Find contours
+    contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    if contours:
+        # Find the largest contour
+        largest_contour = max(contours, key=cv2.contourArea)
+        x, y, w, h = cv2.boundingRect(largest_contour)
+
+        # Add padding
+        padding = 50
+        height, width = cv_img.shape[:2]
+        x = max(0, x - padding)
+        y = max(0, y - padding)
+        w = min(width - x, w + 2*padding)
+        h = min(height - y, h + 2*padding)
+
+        return cv_img[y:y+h, x:x+w]
+    else:
+        # If no contours found, return center crop
+        height, width = cv_img.shape[:2]
+        center_x = width // 2
+        center_y = height // 2
+        crop_width = width // 3
+        crop_height = height // 3
+        return cv_img[center_y-crop_height//2:center_y+crop_height//2,
+                     center_x-crop_width//2:center_x+crop_width//2]
+
+# Example usage in your Streamlit app:
+def process_panorama(panorama_url):
+    """
+    Process a panoramic image to get a street-centered crop
+    """
+    try:
+        cropped_image = detect_and_crop_street(panorama_url)
+        return cropped_image
+    except Exception as e:
+        st.error(f"Error processing panorama: {str(e)}")
+        return None
+
 def get_bounding_box(lat, lon):
     """
     Create a bounding box around a point that extends roughly 25 meters in each direction
@@ -54,9 +167,8 @@ def get_nearest_image(lat, lon):
     """
     bbox = get_bounding_box(lat, lon)
     params = {
-        'fields': 'id,thumb_1024_url',
+        'fields': 'id,thumb_1024_url,is_pano',
         'limit': 1,
-        'is_pano': False,
         'bbox': f'{bbox[0]},{bbox[1]},{bbox[2]},{bbox[3]}'
     }
 
@@ -69,11 +181,11 @@ def get_nearest_image(lat, lon):
         )
         response.raise_for_status()
         data = response.json()
-        
+
         if 'data' in data and len(data['data']) > 0:
             return data['data'][0]
         return None
-        
+
     except requests.exceptions.RequestException as e:
         st.error(f"Error fetching Mapillary data: {str(e)}")
         return None
@@ -132,7 +244,7 @@ def load_knn():
 
 def main():
     st.title("Percept: Map Explorer")
-    
+
     try:
         with st.spinner('Loading CLIP model... This may take a moment.'):
             model, preprocess, tokenizer = load_model()
@@ -148,35 +260,38 @@ def main():
     # Initialize the map centered on Amsterdam
     amsterdam_coords = [52.3676, 4.9041]
     m = folium.Map(location=amsterdam_coords, zoom_start=13)
-    
+
     # Add a marker for Amsterdam city center
     folium.Marker(
         amsterdam_coords,
         popup="Amsterdam City Center",
         icon=folium.Icon(color="red", icon="info-sign")
     ).add_to(m)
-    
+
     # Display the map and get clicked coordinates
     map_data = st_folium(m, height=400, width=700)
-    
+
     # Check if a location was clicked
     if map_data['last_clicked']:
         lat = map_data['last_clicked']['lat']
         lng = map_data['last_clicked']['lng']
-        
+
         st.write(f"Selected coordinates: {lat:.4f}, {lng:.4f}")
-        
+
         # Get nearest Mapillary image
         with st.spinner('Fetching street view image...'):
             image_data = get_nearest_image(lat, lng)
-            
+
             if image_data:
                 # Display the image
                 try:
-                    response = requests.get(image_data['thumb_1024_url'])
-                    image = Image.open(BytesIO(response.content))
+                    if image_data['is_pano']:
+                        image = process_panorama(image_data['thumb_1024_url'])
+                    else:
+                        response = requests.get(image_data['thumb_1024_url'])
+                        image = Image.open(BytesIO(response.content))
                     st.image(image, caption="Street View", width=400)
-                    
+
                     # Add download button
                     st.download_button(
                         label="Download Image",
@@ -201,7 +316,7 @@ def main():
                         k = 40
                         for cat in categories:
                             st.write(cat, f'rating = {knn_get_score(knn, k, cat, vec):.1f}')
-                    
+
                 except Exception as e:
                     st.error(f"Error displaying image: {str(e)}")
             else:

requirements.txt

@@ -5,3 +5,6 @@ Pillow
 requests
 open_clip_torch
 scikit-learn
+opencv-python-headless
+ultralytics
+numpy