Update app.py

app.py

@@ -1,4 +1,3 @@
-
 import streamlit as st
 import json
 import ee
@@ -229,77 +228,38 @@ def aggregate_data_yearly(collection):
     yearly_images = ee.List(grouped_by_year.map(calculate_yearly_mean))
     return ee.ImageCollection(yearly_images)
 
-# # Cloud percentage calculation
-# def calculate_cloud_percentage(image, cloud_band='QA60'):
-#     qa60 = image.select(cloud_band)
-#     opaque_clouds = qa60.bitwiseAnd(1 << 10)
-#     cirrus_clouds = qa60.bitwiseAnd(1 << 11)
-#     cloud_mask = opaque_clouds.Or(cirrus_clouds)
-#     total_pixels = qa60.reduceRegion(
-#         reducer=ee.Reducer.count(),
-#         geometry=image.geometry(),
-#         scale=60,
-#         maxPixels=1e13
-#     ).get(cloud_band)
-#     cloudy_pixels = cloud_mask.reduceRegion(
-#         reducer=ee.Reducer.sum(),
-#         geometry=image.geometry(),
-#         scale=60,
-#         maxPixels=1e13
-#     ).get(cloud_band)
-#     if total_pixels == 0:
-#         return 0
-#     return ee.Number(cloudy_pixels).divide(ee.Number(total_pixels)).multiply(100)
-
-def calculate_cloud_percentage(image):
-    # Check if QA60 band exists (Level-1C)
-    bands = image.bandNames().getInfo()
-    if 'QA60' in bands:
-        qa60 = image.select('QA60')
-        opaque_clouds = qa60.bitwiseAnd(1 << 10)
-        cirrus_clouds = qa60.bitwiseAnd(1 << 11)
-        cloud_mask = opaque_clouds.Or(cirrus_clouds)
-    elif 'SCL' in bands:  # Check if SCL band exists (Level-2A)
-        scl = image.select('SCL')
-        cloud_mask = scl.eq(8).Or(scl.eq(9)).Or(scl.eq(10))  # Cloud, cirrus, snow/ice
-    else:
-        raise ValueError("Cloud masking bands (QA60 or SCL) not found in the dataset.")
-
-    total_pixels = cloud_mask.reduceRegion(
+# Cloud percentage calculation
+def calculate_cloud_percentage(image, cloud_band='QA60'):
+    qa60 = image.select(cloud_band)
+    opaque_clouds = qa60.bitwiseAnd(1 << 10)
+    cirrus_clouds = qa60.bitwiseAnd(1 << 11)
+    cloud_mask = opaque_clouds.Or(cirrus_clouds)
+    total_pixels = qa60.reduceRegion(
         reducer=ee.Reducer.count(),
         geometry=image.geometry(),
         scale=60,
         maxPixels=1e13
-    ).get('constant')  # Use 'constant' for renamed bands
-
+    ).get(cloud_band)
     cloudy_pixels = cloud_mask.reduceRegion(
         reducer=ee.Reducer.sum(),
         geometry=image.geometry(),
         scale=60,
         maxPixels=1e13
-    ).get('constant')
-
+    ).get(cloud_band)
     if total_pixels == 0:
         return 0
     return ee.Number(cloudy_pixels).divide(ee.Number(total_pixels)).multiply(100)
-    
+
+# Preprocessing function
 def preprocess_collection(collection, pixel_cloud_threshold):
     def mask_cloudy_pixels(image):
-        bands = image.bandNames().getInfo()
-        if 'QA60' in bands:  # Level-1C
-            qa60 = image.select('QA60')
-            opaque_clouds = qa60.bitwiseAnd(1 << 10)
-            cirrus_clouds = qa60.bitwiseAnd(1 << 11)
-            cloud_mask = opaque_clouds.Or(cirrus_clouds)
-        elif 'SCL' in bands:  # Level-2A
-            scl = image.select('SCL')
-            cloud_mask = scl.eq(8).Or(scl.eq(9)).Or(scl.eq(10))  # Cloud, cirrus, snow/ice
-        else:
-            raise ValueError("Cloud masking bands (QA60 or SCL) not found in the dataset.")
-        
+        qa60 = image.select('QA60')
+        opaque_clouds = qa60.bitwiseAnd(1 << 10)
+        cirrus_clouds = qa60.bitwiseAnd(1 << 11)
+        cloud_mask = opaque_clouds.Or(cirrus_clouds)
         clear_pixels = cloud_mask.Not()
         return image.updateMask(clear_pixels)
-
+    
     if pixel_cloud_threshold > 0:
         return collection.map(mask_cloudy_pixels)
     return collection
@@ -329,18 +289,19 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
                 roi = roi.buffer(-30).bounds()
         except ValueError:
             return None
-
+    
     # Filter collection by date and area first
     collection = ee.ImageCollection(dataset_id) \
         .filterDate(ee.Date(start_date_str), ee.Date(end_date_str)) \
         .filterBounds(roi)
-
-    # Apply cloud filtering if applicable
+    
+    st.write(f"After initial filtering: {collection.size().getInfo()} images")
+    
+    # Apply pixel cloud masking if threshold > 0
     if pixel_cloud_threshold > 0:
         collection = preprocess_collection(collection, pixel_cloud_threshold)
         st.write(f"After cloud masking: {collection.size().getInfo()} images")
-
-    # Apply temporal aggregation
+    
     if aggregation_period.lower() == 'custom (start date to end date)':
         collection = aggregate_data_custom(collection)
     elif aggregation_period.lower() == 'daily':
@@ -351,8 +312,7 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
         collection = aggregate_data_monthly(collection, start_date_str, end_date_str)
     elif aggregation_period.lower() == 'yearly':
         collection = aggregate_data_yearly(collection)
-
-    # Process the filtered collection
+    
     image_list = collection.toList(collection.size())
     processed_weeks = set()
     aggregated_results = []
@@ -383,7 +343,7 @@ def process_single_geometry(row, start_date_str, end_date_str, dataset_id, selec
             timestamp = image.get('year')
             period_label = 'Year'
             date = ee.Date(timestamp).format('YYYY').getInfo()
-
+        
         index_image = calculate_custom_formula(image, roi, selected_bands, custom_formula, reducer_choice, dataset_id, user_scale=user_scale)
         try:
             index_value = index_image.reduceRegion(
@@ -415,23 +375,16 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
     progress_bar = st.progress(0)
     progress_text = st.empty()
     start_time = time.time()
-
-    # Fetch the original collection
+    
     raw_collection = ee.ImageCollection(dataset_id) \
         .filterDate(ee.Date(start_date_str), ee.Date(end_date_str))
-
+    
     st.write(f"Original Collection Size: {raw_collection.size().getInfo()}")
-
-    # Apply cloud filtering if applicable
-    if pixel_cloud_threshold > 0:
-        try:
-            raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
-            st.write(f"Filtered Collection Size (After Cloud Masking): {raw_collection.size().getInfo()}")
-        except Exception as e:
-            st.error(f"Cloud masking failed: {str(e)}")
-            st.stop()
-
-    # Use ThreadPoolExecutor to process each geometry
+    
+    if tile_cloud_threshold > 0 or pixel_cloud_threshold > 0:
+        raw_collection = preprocess_collection(raw_collection, pixel_cloud_threshold)
+        st.write(f"Preprocessed Collection Size: {raw_collection.size().getInfo()}")
+    
     with ThreadPoolExecutor(max_workers=10) as executor:
         futures = []
         for idx, row in locations_df.iterrows():
@@ -453,7 +406,6 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
                 user_scale=user_scale
             )
             futures.append(future)
-
         completed = 0
         for future in as_completed(futures):
             result = future.result()
@@ -463,10 +415,10 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
             progress_percentage = completed / total_steps
             progress_bar.progress(progress_percentage)
             progress_text.markdown(f"Processing: {int(progress_percentage * 100)}%")
-
+    
     end_time = time.time()
     processing_time = end_time - start_time
-
+    
     if aggregated_results:
         result_df = pd.DataFrame(aggregated_results)
         if aggregation_period.lower() == 'custom (start date to end date)':
@@ -484,8 +436,8 @@ def process_aggregation(locations_df, start_date_str, end_date_str, dataset_id,
         else:
             return result_df.to_dict(orient='records'), processing_time
     return [], processing_time
-    
-#streamlit logic
+
+# Streamlit App Logic
 st.markdown("<h5>Image Collection</h5>", unsafe_allow_html=True)
 imagery_base = st.selectbox("Select Imagery Base", ["Sentinel", "Landsat", "MODIS", "VIIRS", "Custom Input"], index=0)
 data = {}