Update app.py

app.py

@@ -1,4 +1,3 @@
-
 import streamlit as st
 import json
 import ee
@@ -118,7 +117,6 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
             band_scales.append(band_scale)
         default_scale = min(band_scales) if band_scales else 30  # Default to 30m if no bands are found
         scale = user_scale if user_scale is not None else default_scale
-        
         # Rescale all bands to the chosen scale
         rescaled_bands = {}
         for band in selected_bands:
@@ -132,7 +130,6 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
                 rescaled_bands[band] = rescaled_band
             else:
                 rescaled_bands[band] = band_image
-        
         # Validate and extract band values
         reduced_values = {}
         reducer = get_reducer(reducer_choice)
@@ -143,13 +140,11 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
                 scale=scale
             ).get(band).getInfo()
             reduced_values[band] = float(value if value is not None else 0)
-        
         # Evaluate the custom formula
         formula = custom_formula
         for band in selected_bands:
             formula = formula.replace(band, str(reduced_values[band]))
         result = eval(formula, {"__builtins__": {}}, reduced_values)
-        
         # Validate the result
         if not isinstance(result, (int, float)):
             raise ValueError("Formula did not result in a numeric value.")
@@ -167,68 +162,6 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
         st.error(f"Unexpected error: {e}")
         return ee.Image(0).rename('custom_result').set('error', str(e))
 
-# Aggregation functions
-def aggregate_data_custom(collection):
-    collection = collection.map(lambda image: image.set('day', ee.Date(image.get('system:time_start')).format('YYYY-MM-dd')))
-    grouped_by_day = collection.aggregate_array('day').distinct()
-    def calculate_daily_mean(day):
-        daily_collection = collection.filter(ee.Filter.eq('day', day))
-        daily_mean = daily_collection.mean()
-        return daily_mean.set('day', day)
-    daily_images = ee.List(grouped_by_day.map(calculate_daily_mean))
-    return ee.ImageCollection(daily_images)
-
-def aggregate_data_daily(collection):
-    def set_day_start(image):
-        date = ee.Date(image.get('system:time_start'))
-        day_start = date.format('YYYY-MM-dd')
-        return image.set('day_start', day_start)
-    collection = collection.map(set_day_start)
-    grouped_by_day = collection.aggregate_array('day_start').distinct()
-    def calculate_daily_mean(day_start):
-        daily_collection = collection.filter(ee.Filter.eq('day_start', day_start))
-        daily_mean = daily_collection.mean()
-        return daily_mean.set('day_start', day_start)
-    daily_images = ee.List(grouped_by_day.map(calculate_daily_mean))
-    return ee.ImageCollection(daily_images)
-
-def aggregate_data_weekly(collection, start_date_str, end_date_str):
-    start_date = ee.Date(start_date_str)
-    end_date = ee.Date(end_date_str)
-    days_diff = end_date.difference(start_date, 'day')
-    num_weeks = days_diff.divide(7).ceil().getInfo()
-    weekly_images = []
-    for week in range(num_weeks):
-        week_start = start_date.advance(week * 7, 'day')
-        week_end = week_start.advance(7, 'day')
-        weekly_collection = collection.filterDate(week_start, week_end)
-        if weekly_collection.size().getInfo() > 0:
-            weekly_mean = weekly_collection.mean()
-            weekly_mean = weekly_mean.set('week_start', week_start.format('YYYY-MM-dd'))
-            weekly_images.append(weekly_mean)
-    return ee.ImageCollection.fromImages(weekly_images)
-
-def aggregate_data_monthly(collection, start_date, end_date):
-    collection = collection.filterDate(start_date, end_date)
-    collection = collection.map(lambda image: image.set('month', ee.Date(image.get('system:time_start')).format('YYYY-MM')))
-    grouped_by_month = collection.aggregate_array('month').distinct()
-    def calculate_monthly_mean(month):
-        monthly_collection = collection.filter(ee.Filter.eq('month', month))
-        monthly_mean = monthly_collection.mean()
-        return monthly_mean.set('month', month)
-    monthly_images = ee.List(grouped_by_month.map(calculate_monthly_mean))
-    return ee.ImageCollection(monthly_images)
-
-def aggregate_data_yearly(collection):
-    collection = collection.map(lambda image: image.set('year', ee.Date(image.get('system:time_start')).format('YYYY')))
-    grouped_by_year = collection.aggregate_array('year').distinct()
-    def calculate_yearly_mean(year):
-        yearly_collection = collection.filter(ee.Filter.eq('year', year))
-        yearly_mean = yearly_collection.mean()
-        return yearly_mean.set('year', year)
-    yearly_images = ee.List(grouped_by_year.map(calculate_yearly_mean))
-    return ee.ImageCollection(yearly_images)
-
 # Cloud percentage calculation
 def calculate_cloud_percentage(image, cloud_band='QA60'):
     qa60 = image.select(cloud_band)
@@ -260,7 +193,6 @@ def preprocess_collection(collection, pixel_cloud_threshold):
         cloud_mask = opaque_clouds.Or(cirrus_clouds)
         clear_pixels = cloud_mask.Not()
         return image.updateMask(clear_pixels)
-    
     if pixel_cloud_threshold > 0:
         return collection.map(mask_cloudy_pixels)
     return collection
@@ -290,20 +222,15 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
                 roi = roi.buffer(-30).bounds()
         except ValueError:
             return None
-    
     # Filter collection by date and area first
-    # Apply spatial filtering
     collection = ee.ImageCollection(dataset_id) \
         .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
         .filterBounds(roi)
-    
-    # Apply cloud filtering if applicable
+    st.write(f"After initial filtering: {collection.size().getInfo()} images")
+    # Apply pixel cloud masking if threshold > 0
     if pixel_cloud_threshold > 0:
         collection = preprocess_collection(collection, pixel_cloud_threshold)
         st.write(f"After cloud masking: {collection.size().getInfo()} images")
-
-     st.write(f"After initial filtering: {collection.size().getInfo()} images")
-    
     if aggregation_period.lower() == 'custom (start date to end date)':
         collection = aggregate_data_custom(collection)
     elif aggregation_period.lower() == 'daily':
@@ -314,7 +241,6 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
         collection = aggregate_data_monthly(collection, start_date_str, end_date_str)
     elif aggregation_period.lower() == 'yearly':
         collection = aggregate_data_yearly(collection)
-    
     image_list = collection.toList(collection.size())
     processed_weeks = set()
     aggregated_results = []
@@ -345,7 +271,6 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
             timestamp = image.get('year')
             period_label = 'Year'
             date = ee.Date(timestamp).format('YYYY').getInfo()
-        
         index_image = calculate_custom_formula(image, roi, selected_bands, custom_formula, reducer_choice, dataset_id, user_scale=user_scale)
         try:
             index_value = index_image.reduceRegion(
@@ -377,40 +302,22 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
     progress_bar = st.progress(0)
     progress_text = st.empty()
     start_time = time.time()
+    raw_collection = ee.ImageCollection(dataset_id) \
+        .filterDate(ee.Date(start_date_str), ee.Date(end_date_str))
     
+    # Log the original collection size
+    st.write(f"Original Collection Size: {raw_collection.size().getInfo()}")
+
     # Apply spatial filtering
     if roi is not None:
-        raw_collection = ee.ImageCollection(dataset_id) \
-            .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
-            .filterBounds(roi)
-    else:
-        raw_collection = ee.ImageCollection(dataset_id) \
-            .filterDate(ee.Date(start_date_str), ee.Date(end_date_str))
-    
-    st.write(f"Original Collection Size: {raw_collection.size().getInfo()}")
-    st.write(f"Original Collection Size: {ee.ImageCollection(dataset_id).filterDate(ee.Date(start_date_str), ee.Date(end_date_str)).size().getInfo()}")
-    st.write(f"Filtered Collection Size (After Spatial Filtering): {raw_collection.size().getInfo()}")
-    
+        raw_collection = raw_collection.filterBounds(roi)
+        st.write(f"Filtered Collection Size (After Spatial Filtering): {raw_collection.size().getInfo()}")
+
+    # Apply cloud masking if threshold > 0
     if pixel_cloud_threshold > 0:
         raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
         st.write(f"Filtered Collection Size (After Cloud Masking): {raw_collection.size().getInfo()}")
-        
-    # if tile_cloud_threshold > 0 or pixel_cloud_threshold > 0:
-    #     raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
-    #     st.write(f"Preprocessed Collection Size: {raw_collection.size().getInfo()}")
-    # Apply cloud filtering
-    if imagery_base == "Sentinel" and "Sentinel-2" in sub_options[sub_selection]:
-        pixel_cloud_threshold = st.slider(
-            "Select Maximum Pixel-Based Cloud Coverage Threshold (%)",
-            min_value=0,
-            max_value=100,
-            value=5,
-            step=5,
-            help="Individual pixels with cloud coverage exceeding this threshold will be masked."
-        )
-        raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
-        st.write(f"Filtered Collection Size (After Cloud Masking): {raw_collection.size().getInfo()}")
-    
+
     with ThreadPoolExecutor(max_workers=10) as executor:
         futures = []
         for idx, row in locations_df.iterrows():
@@ -441,10 +348,8 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
             progress_percentage = completed / total_steps
             progress_bar.progress(progress_percentage)
             progress_text.markdown(f"Processing: {int(progress_percentage * 100)}%")
-    
     end_time = time.time()
     processing_time = end_time - start_time
-    
     if aggregated_results:
         result_df = pd.DataFrame(aggregated_results)
         if aggregation_period.lower() == 'custom (start date to end date)':
@@ -526,7 +431,6 @@ elif imagery_base == "Custom Input":
 if not data:
     st.error("No valid dataset available. Please check your inputs.")
     st.stop()
-
 st.markdown("<hr><h5><b>{}</b></h5>".format(imagery_base), unsafe_allow_html=True)
 main_selection = st.selectbox(f"Select {imagery_base} Dataset Category", list(data.keys()))
 sub_selection = None
@@ -546,13 +450,10 @@ if main_selection:
             st.write(f"Default Scale for Selected Dataset: {default_scale} meters")
         except Exception as e:
             st.error(f"Error fetching default scale: {str(e)}")
-
 st.markdown("<hr><h5><b>Earth Engine Index Calculator</b></h5>", unsafe_allow_html=True)
 if main_selection and sub_selection:
     dataset_bands = data[main_selection]["bands"].get(sub_selection, [])
     st.write(f"Available Bands for {sub_options[sub_selection]}: {', '.join(dataset_bands)}")
-    
-
     # Fetch nominal scales for all bands in the selected dataset
     if dataset_id:
         try:
@@ -560,20 +461,16 @@ if main_selection and sub_selection:
             collection = ee.ImageCollection(dataset_id)
             first_image = collection.first()
             band_names = first_image.bandNames().getInfo()
-    
             # Extract scales for all bands
             band_scales = []
             for band in band_names:
                 band_scale = first_image.select(band).projection().nominalScale().getInfo()
                 band_scales.append(band_scale)
-    
             # Identify unique scales using np.unique
             unique_scales = np.unique(band_scales)
-    
             # Display the unique scales to the user
             st.write(f"Nominal Scales for Bands: {band_scales}")
             st.write(f"Unique Scales in Dataset: {unique_scales}")
-    
             # If there are multiple unique scales, allow the user to choose one
             if len(unique_scales) > 1:
                 selected_scale = st.selectbox(
@@ -586,11 +483,9 @@ if main_selection and sub_selection:
             else:
                 default_scale = unique_scales[0]
                 st.write(f"Default Scale for Dataset: {default_scale} meters")
-    
         except Exception as e:
             st.error(f"Error fetching band scales: {str(e)}")
             default_scale = 30  # Fallback to 30 meters if an error occurs
-        
     selected_bands = st.multiselect(
         "Select 1 or 2 Bands for Calculation",
         options=dataset_bands,
@@ -629,18 +524,15 @@ if main_selection and sub_selection:
             st.warning("Please enter a custom formula to proceed.")
             st.stop()
         st.write(f"Custom Formula: {custom_formula}")
-
 reducer_choice = st.selectbox(
     "Select Reducer (e.g, mean , sum , median , min , max , count)",
     ['mean', 'sum', 'median', 'min', 'max', 'count'],
     index=0
 )
-
 start_date = st.date_input("Start Date", value=pd.to_datetime('2024-11-01'))
 end_date = st.date_input("End Date", value=pd.to_datetime('2024-12-01'))
 start_date_str = start_date.strftime('%Y-%m-%d')
 end_date_str = end_date.strftime('%Y-%m-%d')
-
 if imagery_base == "Sentinel" and "Sentinel-2" in sub_options[sub_selection]:
     st.markdown("<h5>Cloud Filtering</h5>", unsafe_allow_html=True)
     pixel_cloud_threshold = st.slider(
@@ -651,17 +543,14 @@ if imagery_base == "Sentinel" and "Sentinel-2" in sub_options[sub_selection]:
         step=5,
         help="Individual pixels with cloud coverage exceeding this threshold will be masked."
     )
-
 aggregation_period = st.selectbox(
     "Select Aggregation Period (e.g, Custom(Start Date to End Date) , Daily , Weekly , Monthly , Yearly)",
     ["Custom (Start Date to End Date)", "Daily", "Weekly", "Monthly", "Yearly"],
     index=0
 )
-
 shape_type = st.selectbox("Do you want to process 'Point' or 'Polygon' data?", ["Point", "Polygon"])
 kernel_size = None
 include_boundary = None
-
 if shape_type.lower() == "point":
     kernel_size = st.selectbox(
         "Select Calculation Area(e.g, Point , 3x3 Kernel , 5x5 Kernel)",
@@ -675,7 +564,6 @@ elif shape_type.lower() == "polygon":
         value=True,
         help="Check to include pixels on the polygon boundary; uncheck to exclude them."
     )
-
 # st.markdown("<h5>Calculation Scale</h5>", unsafe_allow_html=True)
 # default_scale = ee.ImageCollection(dataset_id).first().select(0).projection().nominalScale().getInfo()
 # user_scale = st.number_input(
@@ -684,7 +572,6 @@ elif shape_type.lower() == "polygon":
 #     value=float(default_scale),
 #     help=f"Default scale for this dataset is {default_scale} meters. Adjust if needed."
 # )
-
 st.markdown("<h5>Calculation Scale</h5>", unsafe_allow_html=True)
 user_scale = st.number_input(
     "Enter Calculation Scale (meters) [Leave blank to use dataset's default scale]",
@@ -692,12 +579,10 @@ user_scale = st.number_input(
     value=float(default_scale),
     help=f"Default scale for this dataset is {default_scale} meters. Adjust if needed."
 )
-
 file_upload = st.file_uploader(f"Upload your {shape_type} data (CSV, GeoJSON, KML)", type=["csv", "geojson", "kml"])
 locations_df = pd.DataFrame()
 original_lat_col = None
 original_lon_col = None
-
 if file_upload is not None:
     if shape_type.lower() == "point":
         if file_upload.name.endswith('.csv'):
@@ -812,7 +697,6 @@ if file_upload is not None:
                     m.add_gdf(gdf=gdf, layer_name=row.get('name', 'Unnamed Polygon'))
             st.write("Map of Uploaded Polygons:")
             m.to_streamlit()
-
 if st.button(f"Calculate {custom_formula}"):
     if not locations_df.empty:
         with st.spinner("Processing Data..."):