Update app.py

app.py

@@ -175,6 +175,7 @@ def aggregate_data_daily(collection):
     daily_images = ee.List(grouped_by_day.map(calculate_daily_mean))
     return ee.ImageCollection(daily_images)
 
+
 def aggregate_data_weekly(collection, start_date_str, end_date_str):
     """
     Aggregates data on a weekly basis, starting from the exact start date provided by the user.
@@ -199,6 +200,7 @@ def aggregate_data_weekly(collection, start_date_str, end_date_str):
     
     return ee.ImageCollection.fromImages(weekly_images)
 
+
 def aggregate_data_monthly(collection, start_date, end_date):
     collection = collection.filterDate(start_date, end_date)
     collection = collection.map(lambda image: image.set('month', ee.Date(image.get('system:time_start')).format('YYYY-MM')))
@@ -210,6 +212,7 @@ def aggregate_data_monthly(collection, start_date, end_date):
     monthly_images = ee.List(grouped_by_month.map(calculate_monthly_mean))
     return ee.ImageCollection(monthly_images)
 
+
 def aggregate_data_yearly(collection):
     collection = collection.map(lambda image: image.set('year', ee.Date(image.get('system:time_start')).format('YYYY')))
     grouped_by_year = collection.aggregate_array('year').distinct()
@@ -252,6 +255,7 @@ def calculate_cloud_percentage(image, cloud_band='QA60'):
         return 0  # Avoid division by zero
     return ee.Number(cloudy_pixels).divide(ee.Number(total_pixels)).multiply(100).getInfo()
 
+# Preprocessing function with cloud filtering
 def preprocess_collection(collection, cloud_threshold):
     """
     Apply cloud filtering to the image collection using the QA60 bitmask.
@@ -283,109 +287,106 @@ def preprocess_collection(collection, cloud_threshold):
     masked_collection = filtered_collection.map(mask_cloudy_pixels)
     return masked_collection
 
-def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selected_bands, reducer_choice, shape_type, aggregation_period, custom_formula, original_lat_col, original_lon_col, kernel_size=None, include_boundary=None, cloud_threshold=0):
-    try:
-        if shape_type.lower() == "point":
-            latitude = row.get('latitude')
-            longitude = row.get('longitude')
-            if pd.isna(latitude) or pd.isna(longitude):
-                return None  # Skip invalid points
-            location_name = row.get('name', f"Location_{row.name}")
-            if kernel_size == "3x3 Kernel":
-                buffer_size = 45  # 90m x 90m
-                roi = ee.Geometry.Point([longitude, latitude]).buffer(buffer_size).bounds()
-            elif kernel_size == "5x5 Kernel":
-                buffer_size = 75  # 150m x 150m
-                roi = ee.Geometry.Point([longitude, latitude]).buffer(buffer_size).bounds()
-            else:  # Point
-                roi = ee.Geometry.Point([longitude, latitude])
-        elif shape_type.lower() == "polygon":
-            polygon_geometry = row.get('geometry')
-            location_name = row.get('name', f"Polygon_{row.name}")
+# Worker function for processing a single geometry
+def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selected_bands, reducer_choice, shape_type, aggregation_period, custom_formula, original_lat_col, original_lon_col, kernel_size=None, include_boundary=None):
+    if shape_type.lower() == "point":
+        latitude = row.get('latitude')
+        longitude = row.get('longitude')
+        if pd.isna(latitude) or pd.isna(longitude):
+            return None  # Skip invalid points
+        location_name = row.get('name', f"Location_{row.name}")
+        if kernel_size == "3x3 Kernel":
+            buffer_size = 45  # 90m x 90m
+            roi = ee.Geometry.Point([longitude, latitude]).buffer(buffer_size).bounds()
+        elif kernel_size == "5x5 Kernel":
+            buffer_size = 75  # 150m x 150m
+            roi = ee.Geometry.Point([longitude, latitude]).buffer(buffer_size).bounds()
+        else:  # Point
+            roi = ee.Geometry.Point([longitude, latitude])
+    elif shape_type.lower() == "polygon":
+        polygon_geometry = row.get('geometry')
+        location_name = row.get('name', f"Polygon_{row.name}")
+        try:
             roi = convert_to_ee_geometry(polygon_geometry)
             if not include_boundary:
                 roi = roi.buffer(-30).bounds()
+        except ValueError:
+            return None  # Skip invalid polygons
 
-        # Filter and aggregate the image collection
-        collection = ee.ImageCollection(dataset_id) \
-            .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
-            .filterBounds(roi)
-        
-        # Apply cloud filtering if threshold > 0
-        if cloud_threshold > 0:
-            collection = preprocess_collection(collection, cloud_threshold)
-        
-        # Apply aggregation
+    # Filter and aggregate the image collection
+    collection = ee.ImageCollection(dataset_id) \
+        .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
+        .filterBounds(roi)
+    
+    if aggregation_period.lower() == 'custom (start date to end date)':
+        collection = aggregate_data_custom(collection)
+    elif aggregation_period.lower() == 'daily':
+        collection = aggregate_data_daily(collection)
+    elif aggregation_period.lower() == 'weekly':
+        collection = aggregate_data_weekly(collection, start_date_str, end_date_str)
+    elif aggregation_period.lower() == 'monthly':
+        collection = aggregate_data_monthly(collection, start_date_str, end_date_str)
+    elif aggregation_period.lower() == 'yearly':
+        collection = aggregate_data_yearly(collection)
+
+    # Process each image in the collection
+    image_list = collection.toList(collection.size())
+    processed_weeks = set()
+    aggregated_results = []
+    for i in range(image_list.size().getInfo()):
+        image = ee.Image(image_list.get(i))
         if aggregation_period.lower() == 'custom (start date to end date)':
-            collection = aggregate_data_custom(collection)
+            timestamp = image.get('day')
+            period_label = 'Date'
+            date = ee.Date(timestamp).format('YYYY-MM-dd').getInfo()
         elif aggregation_period.lower() == 'daily':
-            collection = aggregate_data_daily(collection)
+            timestamp = image.get('day_start')
+            period_label = 'Date'
+            date = ee.String(timestamp).getInfo()
         elif aggregation_period.lower() == 'weekly':
-            collection = aggregate_data_weekly(collection, start_date_str, end_date_str)
+            timestamp = image.get('week_start')
+            period_label = 'Week'
+            date = ee.String(timestamp).getInfo()
+            if (pd.to_datetime(date) < pd.to_datetime(start_date_str) or 
+                pd.to_datetime(date) > pd.to_datetime(end_date_str) or 
+                date in processed_weeks):
+                continue
+            processed_weeks.add(date)
         elif aggregation_period.lower() == 'monthly':
-            collection = aggregate_data_monthly(collection, start_date_str, end_date_str)
+            timestamp = image.get('month')
+            period_label = 'Month'
+            date = ee.Date(timestamp).format('YYYY-MM').getInfo()
         elif aggregation_period.lower() == 'yearly':
-            collection = aggregate_data_yearly(collection)
+            timestamp = image.get('year')
+            period_label = 'Year'
+            date = ee.Date(timestamp).format('YYYY').getInfo()
 
-        # Get the list of images
-        image_list = collection.toList(collection.size())
-        n_images = image_list.size().getInfo()
-        
-        aggregated_results = []
-        for i in range(n_images):
-            image = ee.Image(image_list.get(i))
-            
-            # Get the timestamp based on aggregation period
-            if aggregation_period.lower() == 'custom (start date to end date)':
-                timestamp = image.get('day')
-                period_label = 'Date'
-                date = ee.Date(timestamp).format('YYYY-MM-dd').getInfo()
-            elif aggregation_period.lower() == 'daily':
-                timestamp = image.get('day_start')
-                period_label = 'Date'
-                date = ee.String(timestamp).getInfo()
-            elif aggregation_period.lower() == 'weekly':
-                timestamp = image.get('week_start')
-                period_label = 'Week'
-                date = ee.String(timestamp).getInfo()
-            elif aggregation_period.lower() == 'monthly':
-                timestamp = image.get('month')
-                period_label = 'Month'
-                date = ee.Date(timestamp).format('YYYY-MM').getInfo()
-            elif aggregation_period.lower() == 'yearly':
-                timestamp = image.get('year')
-                period_label = 'Year'
-                date = ee.Date(timestamp).format('YYYY').getInfo()
-
-            # Calculate the custom formula
-            index_image = calculate_custom_formula(image, roi, selected_bands, custom_formula, reducer_choice, scale=30)
-            
-            # Get the reduced value
+        index_image = calculate_custom_formula(image, roi, selected_bands, custom_formula, reducer_choice, scale=30)
+        try:
             index_value = index_image.reduceRegion(
                 reducer=get_reducer(reducer_choice),
                 geometry=roi,
                 scale=30
-            ).get('custom_result').getInfo()
-            
-            if isinstance(index_value, (int, float)):
+            ).get('custom_result')
+            calculated_value = index_value.getInfo()
+            if isinstance(calculated_value, (int, float)):
                 result = {
                     'Location Name': location_name,
                     period_label: date,
                     'Start Date': start_date_str,
                     'End Date': end_date_str,
-                    'Calculated Value': index_value
+                    'Calculated Value': calculated_value
                 }
                 if shape_type.lower() == 'point':
-                    result[original_lat_col] = latitude
-                    result[original_lon_col] = longitude
+                    result[original_lat_col] = latitude  # Use original column name
+                    result[original_lon_col] = longitude  # Use original column name
                 aggregated_results.append(result)
-                
-        return aggregated_results
-        
-    except Exception as e:
-        st.error(f"Error processing {location_name if 'location_name' in locals() else 'location'}: {str(e)}")
-        return None
+        except Exception as e:
+            st.error(f"Error retrieving value for {location_name}: {e}")
+    return aggregated_results
+
 
+# Main processing function
 def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id, selected_bands, reducer_choice, shape_type, aggregation_period, original_lat_col, original_lon_col, custom_formula="", kernel_size=None, include_boundary=None, cloud_threshold=0):
     aggregated_results = []
     total_steps = len(locations_df)
@@ -393,6 +394,20 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
     progress_text = st.empty()
     start_time = time.time()  # Start timing the process
     
+    # Preprocess the image collection with cloud filtering
+    raw_collection = ee.ImageCollection(dataset_id) \
+        .filterDate(ee.Date(start_date_str), ee.Date(end_date_str))
+
+    # Print the size of the original collection
+    st.write(f"Original Collection Size: {raw_collection.size().getInfo()}")
+    
+    # Apply cloud filtering if threshold > 0
+    if cloud_threshold > 0:
+        raw_collection = preprocess_collection(raw_collection, cloud_threshold)
+
+        # Print the size of the preprocessed collection
+        st.write(f"Preprocessed Collection Size: {raw_collection.size().getInfo()}")
+    
     with ThreadPoolExecutor(max_workers=10) as executor:
         futures = []
         for idx, row in locations_df.iterrows():
@@ -410,8 +425,7 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
                 original_lat_col,
                 original_lon_col,
                 kernel_size,
-                include_boundary,
-                cloud_threshold
+                include_boundary
             )
             futures.append(future)
         completed = 0
@@ -435,6 +449,7 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
                 'Start Date': 'first',
                 'End Date': 'first',
                 'Calculated Value': 'mean'
+                # 'Date Range': 'first'  # Include Date Range in aggregation
             }
             if shape_type.lower() == 'point':
                 agg_dict[original_lat_col] = 'first'
@@ -578,6 +593,7 @@ cloud_threshold = st.slider(
     help="Tiles with cloud coverage exceeding this threshold will be excluded. Individual cloudy pixels will also be masked."
 )
 
+
 aggregation_period = st.selectbox(
     "Select Aggregation Period (e.g, Custom(Start Date to End Date) , Daily , Weekly , Monthly , Yearly)",
     ["Custom (Start Date to End Date)", "Daily", "Weekly", "Monthly", "Yearly"],
@@ -730,113 +746,44 @@ if file_upload is not None:
             st.write("Map of Uploaded Polygons:")
             m.to_streamlit()
 
-def visualize_clouds(collection, roi, cloud_band='QA60'):
-    """
-    Visualize cloud coverage in the selected region of interest (ROI).
-    """
-    # Select the first image in the collection for visualization
-    first_image = ee.Image(collection.first())
-    
-    # Decode the QA60 bitmask for clouds
-    qa60 = first_image.select(cloud_band)
-    opaque_clouds = qa60.bitwiseAnd(1 << 10)  # Bit 10: Opaque clouds
-    cirrus_clouds = qa60.bitwiseAnd(1 << 11)  # Bit 11: Cirrus clouds
-    cloud_mask = opaque_clouds.Or(cirrus_clouds)
-    
-    # Create a cloud visualization layer
-    cloud_vis = {
-        'min': 0,
-        'max': 1,
-        'palette': ['white', 'blue']  # White = clear, Blue = cloudy
-    }
-    
-    return cloud_mask, cloud_vis
-
-# In the cloud preview button section:
-def visualize_clouds(collection, roi, cloud_band='QA60'):
-    """
-    Visualize cloud coverage in the selected region of interest (ROI).
-    """
-    # Select the first image in the collection for visualization
-    first_image = ee.Image(collection.first())
-    
-    # Decode the QA60 bitmask for clouds
-    qa60 = first_image.select(cloud_band)
-    opaque_clouds = qa60.bitwiseAnd(1 << 10)  # Bit 10: Opaque clouds
-    cirrus_clouds = qa60.bitwiseAnd(1 << 11)  # Bit 11: Cirrus clouds
-    cloud_mask = opaque_clouds.Or(cirrus_clouds)
-    
-    # Create a cloud visualization layer
-    cloud_vis = {
-        'min': 0,
-        'max': 1,
-        'palette': ['white', 'blue']  # White = clear, Blue = cloudy
-    }
-    
-    return cloud_mask, cloud_vis
-
-# In the cloud preview button section:
-if st.button("Preview Cloud Coverage"):
-    if not locations_df.empty:
-        with st.spinner("Loading Cloud Coverage Preview..."):
-            try:
-                # Debug prints (properly indented)
-                st.write(f"Leafmap version: {leafmap.__version__}")
-                
-                # Filter the collection for the uploaded geometry
-                if shape_type.lower() == "point":
-                    latitude = locations_df.iloc[0]['latitude']
-                    longitude = locations_df.iloc[0]['longitude']
-                    roi = ee.Geometry.Point([longitude, latitude])
-                elif shape_type.lower() == "polygon":
-                    polygon_geometry = locations_df.iloc[0]['geometry']
-                    roi = convert_to_ee_geometry(polygon_geometry)
-                
-                # Load the image collection
-                raw_collection = ee.ImageCollection(dataset_id) \
-                    .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
-                    .filterBounds(roi)
-                
-                # Debug prints (properly indented)
-                st.write(f"Collection size: {raw_collection.size().getInfo()}")
-                st.write(f"Available bands: {raw_collection.first().bandNames().getInfo()}")
-                
-                if raw_collection.size().getInfo() == 0:
-                    st.warning("No images found for the selected date range and location.")
-                    st.stop()
-                
-                # Visualize cloud coverage
-                cloud_mask, cloud_vis = visualize_clouds(raw_collection, roi, cloud_band='QA60')
-                
-                # Create a new map centered on the ROI
-                centroid = roi.centroid().coordinates().getInfo()
-                m = leafmap.Map(center=[centroid[1], centroid[0]], zoom=10)
-                
-                # Add the layers using the proper Leafmap methods
-                m.add_layer(cloud_mask, cloud_vis, "Cloud Mask")
-                m.add_layer(roi, {'color': 'red'}, "Region of Interest")
-                
-                st.write("Cloud Coverage Preview:")
-                m.to_streamlit()
-            except Exception as e:
-                st.error(f"Error visualizing cloud coverage: {str(e)}")
-    else:
-        st.warning("Please upload a valid file to proceed.")
-
-# After visualizing cloud coverage
-st.markdown("<h5>Adjust Cloud Threshold</h5>", unsafe_allow_html=True)
-new_cloud_threshold = st.slider(
-    "Select Maximum Cloud Coverage Threshold (%)",
-    min_value=0,
-    max_value=50,
-    value=cloud_threshold,  # Start with the current threshold
-    step=5,
-    help="Lower the threshold to exclude more cloudy images."
-)
-
-if st.button("Apply New Cloud Threshold"):
-    cloud_threshold = new_cloud_threshold
-    st.success(f"New cloud threshold applied: {cloud_threshold}%")
+# if st.button(f"Calculate {custom_formula}"):
+#     if not locations_df.empty:
+#         with st.spinner("Processing Data..."):
+#             try:
+#                 results, processing_time = process_aggregation(
+#                     locations_df,
+#                     start_date_str,
+#                     end_date_str,
+#                     dataset_id,
+#                     selected_bands,
+#                     reducer_choice,
+#                     shape_type,
+#                     aggregation_period,
+#                     original_lat_col,
+#                     original_lon_col,
+#                     custom_formula,
+#                     kernel_size,
+#                     include_boundary
+#                 )
+#                 if results:
+#                     result_df = pd.DataFrame(results)
+#                     st.write(f"Processed Results Table ({aggregation_period}) for Formula: {custom_formula}")
+#                     st.dataframe(result_df)
+#                     filename = f"{main_selection}_{dataset_id}_{start_date.strftime('%Y%m%d')}_{end_date.strftime('%Y%m%d')}_{aggregation_period.lower()}.csv"
+#                     st.download_button(
+#                         label="Download results as CSV",
+#                         data=result_df.to_csv(index=False).encode('utf-8'),
+#                         file_name=filename,
+#                         mime='text/csv'
+#                     )
+#                     st.success(f"Processing complete! Total processing time: {processing_time:.2f} seconds.")
+#                 else:
+#                     st.warning("No results were generated. Check your inputs or formula.")
+#                     st.info(f"Total processing time: {processing_time:.2f} seconds.")
+#             except Exception as e:
+#                 st.error(f"An error occurred during processing: {str(e)}")
+#     else:
+#         st.warning("Please upload a valid file to proceed.")
         
 if st.button(f"Calculate {custom_formula}"):
     if not locations_df.empty: