Update app.py

app.py

@@ -1,3 +1,4 @@
+
 import streamlit as st
 import json
 import ee
@@ -117,6 +118,7 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
             band_scales.append(band_scale)
         default_scale = min(band_scales) if band_scales else 30  # Default to 30m if no bands are found
         scale = user_scale if user_scale is not None else default_scale
+        
         # Rescale all bands to the chosen scale
         rescaled_bands = {}
         for band in selected_bands:
@@ -130,6 +132,7 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
                 rescaled_bands[band] = rescaled_band
             else:
                 rescaled_bands[band] = band_image
+        
         # Validate and extract band values
         reduced_values = {}
         reducer = get_reducer(reducer_choice)
@@ -140,11 +143,13 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
                 scale=scale
             ).get(band).getInfo()
             reduced_values[band] = float(value if value is not None else 0)
+        
         # Evaluate the custom formula
         formula = custom_formula
         for band in selected_bands:
             formula = formula.replace(band, str(reduced_values[band]))
         result = eval(formula, {"__builtins__": {}}, reduced_values)
+        
         # Validate the result
         if not isinstance(result, (int, float)):
             raise ValueError("Formula did not result in a numeric value.")
@@ -162,6 +167,68 @@ def calculate_custom_formula(image, geometry, selected_bands, custom_formula, re
         st.error(f"Unexpected error: {e}")
         return ee.Image(0).rename('custom_result').set('error', str(e))
 
+# Aggregation functions
+def aggregate_data_custom(collection):
+    collection = collection.map(lambda image: image.set('day', ee.Date(image.get('system:time_start')).format('YYYY-MM-dd')))
+    grouped_by_day = collection.aggregate_array('day').distinct()
+    def calculate_daily_mean(day):
+        daily_collection = collection.filter(ee.Filter.eq('day', day))
+        daily_mean = daily_collection.mean()
+        return daily_mean.set('day', day)
+    daily_images = ee.List(grouped_by_day.map(calculate_daily_mean))
+    return ee.ImageCollection(daily_images)
+
+def aggregate_data_daily(collection):
+    def set_day_start(image):
+        date = ee.Date(image.get('system:time_start'))
+        day_start = date.format('YYYY-MM-dd')
+        return image.set('day_start', day_start)
+    collection = collection.map(set_day_start)
+    grouped_by_day = collection.aggregate_array('day_start').distinct()
+    def calculate_daily_mean(day_start):
+        daily_collection = collection.filter(ee.Filter.eq('day_start', day_start))
+        daily_mean = daily_collection.mean()
+        return daily_mean.set('day_start', day_start)
+    daily_images = ee.List(grouped_by_day.map(calculate_daily_mean))
+    return ee.ImageCollection(daily_images)
+
+def aggregate_data_weekly(collection, start_date_str, end_date_str):
+    start_date = ee.Date(start_date_str)
+    end_date = ee.Date(end_date_str)
+    days_diff = end_date.difference(start_date, 'day')
+    num_weeks = days_diff.divide(7).ceil().getInfo()
+    weekly_images = []
+    for week in range(num_weeks):
+        week_start = start_date.advance(week * 7, 'day')
+        week_end = week_start.advance(7, 'day')
+        weekly_collection = collection.filterDate(week_start, week_end)
+        if weekly_collection.size().getInfo() > 0:
+            weekly_mean = weekly_collection.mean()
+            weekly_mean = weekly_mean.set('week_start', week_start.format('YYYY-MM-dd'))
+            weekly_images.append(weekly_mean)
+    return ee.ImageCollection.fromImages(weekly_images)
+
+def aggregate_data_monthly(collection, start_date, end_date):
+    collection = collection.filterDate(start_date, end_date)
+    collection = collection.map(lambda image: image.set('month', ee.Date(image.get('system:time_start')).format('YYYY-MM')))
+    grouped_by_month = collection.aggregate_array('month').distinct()
+    def calculate_monthly_mean(month):
+        monthly_collection = collection.filter(ee.Filter.eq('month', month))
+        monthly_mean = monthly_collection.mean()
+        return monthly_mean.set('month', month)
+    monthly_images = ee.List(grouped_by_month.map(calculate_monthly_mean))
+    return ee.ImageCollection(monthly_images)
+
+def aggregate_data_yearly(collection):
+    collection = collection.map(lambda image: image.set('year', ee.Date(image.get('system:time_start')).format('YYYY')))
+    grouped_by_year = collection.aggregate_array('year').distinct()
+    def calculate_yearly_mean(year):
+        yearly_collection = collection.filter(ee.Filter.eq('year', year))
+        yearly_mean = yearly_collection.mean()
+        return yearly_mean.set('year', year)
+    yearly_images = ee.List(grouped_by_year.map(calculate_yearly_mean))
+    return ee.ImageCollection(yearly_images)
+
 # Cloud percentage calculation
 def calculate_cloud_percentage(image, cloud_band='QA60'):
     qa60 = image.select(cloud_band)
@@ -193,6 +260,7 @@ def preprocess_collection(collection, pixel_cloud_threshold):
         cloud_mask = opaque_clouds.Or(cirrus_clouds)
         clear_pixels = cloud_mask.Not()
         return image.updateMask(clear_pixels)
+    
     if pixel_cloud_threshold > 0:
         return collection.map(mask_cloudy_pixels)
     return collection
@@ -222,15 +290,20 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
                 roi = roi.buffer(-30).bounds()
         except ValueError:
             return None
+    
     # Filter collection by date and area first
+    # Apply spatial filtering
     collection = ee.ImageCollection(dataset_id) \
         .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
         .filterBounds(roi)
-    st.write(f"After initial filtering: {collection.size().getInfo()} images")
-    # Apply pixel cloud masking if threshold > 0
+    
+    # Apply cloud filtering if applicable
     if pixel_cloud_threshold > 0:
         collection = preprocess_collection(collection, pixel_cloud_threshold)
         st.write(f"After cloud masking: {collection.size().getInfo()} images")
+
+     st.write(f"After initial filtering: {collection.size().getInfo()} images")
+    
     if aggregation_period.lower() == 'custom (start date to end date)':
         collection = aggregate_data_custom(collection)
     elif aggregation_period.lower() == 'daily':
@@ -241,6 +314,7 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
         collection = aggregate_data_monthly(collection, start_date_str, end_date_str)
     elif aggregation_period.lower() == 'yearly':
         collection = aggregate_data_yearly(collection)
+    
     image_list = collection.toList(collection.size())
     processed_weeks = set()
     aggregated_results = []
@@ -271,6 +345,7 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
             timestamp = image.get('year')
             period_label = 'Year'
             date = ee.Date(timestamp).format('YYYY').getInfo()
+        
         index_image = calculate_custom_formula(image, roi, selected_bands, custom_formula, reducer_choice, dataset_id, user_scale=user_scale)
         try:
             index_value = index_image.reduceRegion(
@@ -295,6 +370,7 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
             st.error(f"Error retrieving value for {location_name}: {e}")
     return aggregated_results
 
+# Process aggregation
 # Process aggregation
 def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id, selected_bands, reducer_choice, shape_type, aggregation_period, original_lat_col, original_lon_col, custom_formula="", kernel_size=None, include_boundary=None, tile_cloud_threshold=0, pixel_cloud_threshold=0, user_scale=None):
     aggregated_results = []
@@ -302,22 +378,18 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
     progress_bar = st.progress(0)
     progress_text = st.empty()
     start_time = time.time()
+
+    # Fetch the original collection size
     raw_collection = ee.ImageCollection(dataset_id) \
         .filterDate(ee.Date(start_date_str), ee.Date(end_date_str))
-    
-    # Log the original collection size
     st.write(f"Original Collection Size: {raw_collection.size().getInfo()}")
 
-    # # Apply spatial filtering
-    # if roi is not None:
-    #     raw_collection = raw_collection.filterBounds(roi)
-    #     st.write(f"Filtered Collection Size (After Spatial Filtering): {raw_collection.size().getInfo()}")
-
-    # Apply cloud masking if threshold > 0
+    # Apply cloud filtering globally if applicable
     if pixel_cloud_threshold > 0:
         raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
         st.write(f"Filtered Collection Size (After Cloud Masking): {raw_collection.size().getInfo()}")
 
+    # Use ThreadPoolExecutor to process each geometry
     with ThreadPoolExecutor(max_workers=10) as executor:
         futures = []
         for idx, row in locations_df.iterrows():
@@ -339,6 +411,7 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
                 user_scale=user_scale
             )
             futures.append(future)
+
         completed = 0
         for future in as_completed(futures):
             result = future.result()
@@ -348,8 +421,10 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
             progress_percentage = completed / total_steps
             progress_bar.progress(progress_percentage)
             progress_text.markdown(f"Processing: {int(progress_percentage * 100)}%")
+
     end_time = time.time()
     processing_time = end_time - start_time
+
     if aggregated_results:
         result_df = pd.DataFrame(aggregated_results)
         if aggregation_period.lower() == 'custom (start date to end date)':
@@ -367,8 +442,8 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
         else:
             return result_df.to_dict(orient='records'), processing_time
     return [], processing_time
-
-# Streamlit App Logic
+    
+#streamlit logic
 st.markdown("<h5>Image Collection</h5>", unsafe_allow_html=True)
 imagery_base = st.selectbox("Select Imagery Base", ["Sentinel", "Landsat", "MODIS", "VIIRS", "Custom Input"], index=0)
 data = {}
@@ -431,6 +506,7 @@ elif imagery_base == "Custom Input":
 if not data:
     st.error("No valid dataset available. Please check your inputs.")
     st.stop()
+
 st.markdown("<hr><h5><b>{}</b></h5>".format(imagery_base), unsafe_allow_html=True)
 main_selection = st.selectbox(f"Select {imagery_base} Dataset Category", list(data.keys()))
 sub_selection = None
@@ -450,10 +526,13 @@ if main_selection:
             st.write(f"Default Scale for Selected Dataset: {default_scale} meters")
         except Exception as e:
             st.error(f"Error fetching default scale: {str(e)}")
+
 st.markdown("<hr><h5><b>Earth Engine Index Calculator</b></h5>", unsafe_allow_html=True)
 if main_selection and sub_selection:
     dataset_bands = data[main_selection]["bands"].get(sub_selection, [])
     st.write(f"Available Bands for {sub_options[sub_selection]}: {', '.join(dataset_bands)}")
+    
+
     # Fetch nominal scales for all bands in the selected dataset
     if dataset_id:
         try:
@@ -461,16 +540,20 @@ if main_selection and sub_selection:
             collection = ee.ImageCollection(dataset_id)
             first_image = collection.first()
             band_names = first_image.bandNames().getInfo()
+    
             # Extract scales for all bands
             band_scales = []
             for band in band_names:
                 band_scale = first_image.select(band).projection().nominalScale().getInfo()
                 band_scales.append(band_scale)
+    
             # Identify unique scales using np.unique
             unique_scales = np.unique(band_scales)
+    
             # Display the unique scales to the user
             st.write(f"Nominal Scales for Bands: {band_scales}")
             st.write(f"Unique Scales in Dataset: {unique_scales}")
+    
             # If there are multiple unique scales, allow the user to choose one
             if len(unique_scales) > 1:
                 selected_scale = st.selectbox(
@@ -483,9 +566,11 @@ if main_selection and sub_selection:
             else:
                 default_scale = unique_scales[0]
                 st.write(f"Default Scale for Dataset: {default_scale} meters")
+    
         except Exception as e:
             st.error(f"Error fetching band scales: {str(e)}")
             default_scale = 30  # Fallback to 30 meters if an error occurs
+        
     selected_bands = st.multiselect(
         "Select 1 or 2 Bands for Calculation",
         options=dataset_bands,
@@ -524,15 +609,18 @@ if main_selection and sub_selection:
             st.warning("Please enter a custom formula to proceed.")
             st.stop()
         st.write(f"Custom Formula: {custom_formula}")
+
 reducer_choice = st.selectbox(
     "Select Reducer (e.g, mean , sum , median , min , max , count)",
     ['mean', 'sum', 'median', 'min', 'max', 'count'],
     index=0
 )
+
 start_date = st.date_input("Start Date", value=pd.to_datetime('2024-11-01'))
 end_date = st.date_input("End Date", value=pd.to_datetime('2024-12-01'))
 start_date_str = start_date.strftime('%Y-%m-%d')
 end_date_str = end_date.strftime('%Y-%m-%d')
+
 if imagery_base == "Sentinel" and "Sentinel-2" in sub_options[sub_selection]:
     st.markdown("<h5>Cloud Filtering</h5>", unsafe_allow_html=True)
     pixel_cloud_threshold = st.slider(
@@ -543,14 +631,17 @@ if imagery_base == "Sentinel" and "Sentinel-2" in sub_options[sub_selection]:
         step=5,
         help="Individual pixels with cloud coverage exceeding this threshold will be masked."
     )
+
 aggregation_period = st.selectbox(
     "Select Aggregation Period (e.g, Custom(Start Date to End Date) , Daily , Weekly , Monthly , Yearly)",
     ["Custom (Start Date to End Date)", "Daily", "Weekly", "Monthly", "Yearly"],
     index=0
 )
+
 shape_type = st.selectbox("Do you want to process 'Point' or 'Polygon' data?", ["Point", "Polygon"])
 kernel_size = None
 include_boundary = None
+
 if shape_type.lower() == "point":
     kernel_size = st.selectbox(
         "Select Calculation Area(e.g, Point , 3x3 Kernel , 5x5 Kernel)",
@@ -564,6 +655,7 @@ elif shape_type.lower() == "polygon":
         value=True,
         help="Check to include pixels on the polygon boundary; uncheck to exclude them."
     )
+
 # st.markdown("<h5>Calculation Scale</h5>", unsafe_allow_html=True)
 # default_scale = ee.ImageCollection(dataset_id).first().select(0).projection().nominalScale().getInfo()
 # user_scale = st.number_input(
@@ -572,6 +664,7 @@ elif shape_type.lower() == "polygon":
 #     value=float(default_scale),
 #     help=f"Default scale for this dataset is {default_scale} meters. Adjust if needed."
 # )
+
 st.markdown("<h5>Calculation Scale</h5>", unsafe_allow_html=True)
 user_scale = st.number_input(
     "Enter Calculation Scale (meters) [Leave blank to use dataset's default scale]",
@@ -579,10 +672,12 @@ user_scale = st.number_input(
     value=float(default_scale),
     help=f"Default scale for this dataset is {default_scale} meters. Adjust if needed."
 )
+
 file_upload = st.file_uploader(f"Upload your {shape_type} data (CSV, GeoJSON, KML)", type=["csv", "geojson", "kml"])
 locations_df = pd.DataFrame()
 original_lat_col = None
 original_lon_col = None
+
 if file_upload is not None:
     if shape_type.lower() == "point":
         if file_upload.name.endswith('.csv'):
@@ -697,6 +792,7 @@ if file_upload is not None:
                     m.add_gdf(gdf=gdf, layer_name=row.get('name', 'Unnamed Polygon'))
             st.write("Map of Uploaded Polygons:")
             m.to_streamlit()
+
 if st.button(f"Calculate {custom_formula}"):
     if not locations_df.empty:
         with st.spinner("Processing Data..."):