feat: extract dataset manipulation from map features

src/dataset/download.py

@@ -0,0 +1,77 @@
+import streamlit as st
+import time
+import logging
+import pandas as pd
+from datasets import load_dataset
+from datasets import DatasetDict, Dataset
+
+############################################################
+# the dataset of observations (hf dataset in our space)
+dataset_id = "Saving-Willy/temp_dataset"
+data_files = "data/train-00000-of-00001.parquet"
+############################################################
+
+m_logger = logging.getLogger(__name__)
+# we can set the log level locally for funcs in this module
+#g_m_logger.setLevel(logging.DEBUG)
+m_logger.setLevel(logging.INFO)
+
+presentation_data_schema = {
+    'lat': 'float',
+    'lon': 'float',
+    'species': 'str',
+}
+
+def try_download_dataset(dataset_id:str, data_files:str) -> dict:
+    """
+    Attempts to download a dataset from Hugging Face, catching any errors that occur.
+    
+    Args:
+        dataset_id (str): The ID of the dataset to download.
+        data_files (str): The data files associated with the dataset.
+    Returns:
+        dict: A dictionary containing the dataset metadata if the download is successful, 
+              or an empty dictionary if an error occurs.
+
+    """
+
+    m_logger.info(f"Starting to download dataset {dataset_id} from Hugging Face")
+    t1 = time.time()
+    try:
+        metadata:DatasetDict = load_dataset(dataset_id, data_files=data_files)
+        t2 = time.time(); elap = t2 - t1
+    except ValueError as e:
+        t2 = time.time(); elap = t2 - t1
+        msg = f"Error downloading dataset: {e}.  (after {elap:.2f}s)."
+        st.error(msg)
+        m_logger.error(msg)
+        metadata = {}
+    except Exception as e:
+        # catch all (other) exceptions and log them, handle them once isolated 
+        t2 = time.time(); elap = t2 - t1
+        msg = f"!!Unknown Error!! downloading dataset: {e}.  (after {elap:.2f}s)."
+        st.error(msg)
+        m_logger.error(msg)
+        metadata = {}
+        
+
+    msg = f"Downloaded dataset: (after {elap:.2f}s). "
+    m_logger.info(msg)
+    st.write(msg)
+    return metadata
+
+def get_dataset():
+    # load/download data from huggingface dataset
+    metadata = try_download_dataset(dataset_id, data_files)
+    
+    if not metadata:
+        # create an empty, but compliant dataframe
+        df = pd.DataFrame(columns=presentation_data_schema).astype(presentation_data_schema)
+    else:
+        # make a pandas df that is compliant with folium/streamlit maps
+        df = pd.DataFrame({
+            'lat': metadata["train"]["latitude"],
+            'lon': metadata["train"]["longitude"],
+            'species': metadata["train"]["selected_class"],}
+        )
+    return df

src/maps/obs_map.py

@@ -1,18 +1,13 @@
 from typing import Tuple
 import logging
 
-import pandas as pd
-from datasets import load_dataset
-from datasets import DatasetDict, Dataset
-
-import time
-
 import streamlit as st
 import folium
 from streamlit_folium import st_folium
 
 import whale_viewer as viewer
 from utils.fix_tabrender import js_show_zeroheight_iframe
+from dataset.download import get_dataset
 
 m_logger = logging.getLogger(__name__)
 # we can set the log level locally for funcs in this module
@@ -66,13 +61,6 @@ _colors = [
 
 whale2color = {k: v for k, v in zip(viewer.WHALE_CLASSES, _colors)}
 
-presentation_data_schema = {
-    'lat': 'float',
-    'lon': 'float',
-    'species': 'str',
-}
-
-
 def create_map(tile_name:str, location:Tuple[float], zoom_start: int = 7) -> folium.Map:
     """
     Create a folium map with the specified tile layer
@@ -124,48 +112,8 @@ def create_map(tile_name:str, location:Tuple[float], zoom_start: int = 7) -> fol
     #folium.LayerControl().add_to(m)
     return m
 
-def try_download_dataset(dataset_id:str, data_files:str) -> dict:
-    """
-    Attempts to download a dataset from Hugging Face, catching any errors that occur.
-    
-    Args:
-        dataset_id (str): The ID of the dataset to download.
-        data_files (str): The data files associated with the dataset.
-    Returns:
-        dict: A dictionary containing the dataset metadata if the download is successful, 
-              or an empty dictionary if an error occurs.
 
-    """
-
-    m_logger.info(f"Starting to download dataset {dataset_id} from Hugging Face")
-    t1 = time.time()
-    try:
-        metadata:DatasetDict = load_dataset(dataset_id, data_files=data_files)
-        t2 = time.time(); elap = t2 - t1
-    except ValueError as e:
-        t2 = time.time(); elap = t2 - t1
-        msg = f"Error downloading dataset: {e}.  (after {elap:.2f}s)."
-        st.error(msg)
-        m_logger.error(msg)
-        metadata = {}
-    except Exception as e:
-        # catch all (other) exceptions and log them, handle them once isolated 
-        t2 = time.time(); elap = t2 - t1
-        msg = f"!!Unknown Error!! downloading dataset: {e}.  (after {elap:.2f}s)."
-        st.error(msg)
-        m_logger.error(msg)
-        metadata = {}
-        
-
-    msg = f"Downloaded dataset: (after {elap:.2f}s). "
-    m_logger.info(msg)
-    st.write(msg)
-    return metadata
-
-
-def present_obs_map(dataset_id:str = "Saving-Willy/Happywhale-kaggle",
-                    data_files:str = "data/train-00000-of-00001.parquet", 
-                    dbg_show_extra:bool = False) -> dict:
+def present_obs_map(dbg_show_extra:bool = False) -> dict:
     """
     Render map plus tile selector, with markers for whale observations
     
@@ -186,19 +134,7 @@ def present_obs_map(dataset_id:str = "Saving-Willy/Happywhale-kaggle",
 
     """
 
-    # load/download data from huggingface dataset
-    metadata = try_download_dataset(dataset_id, data_files)
-    
-    if not metadata:
-        # create an empty, but compliant dataframe
-        _df = pd.DataFrame(columns=presentation_data_schema).astype(presentation_data_schema)
-    else:
-        # make a pandas df that is compliant with folium/streamlit maps
-        _df = pd.DataFrame({
-            'lat': metadata["train"]["latitude"],
-            'lon': metadata["train"]["longitude"],
-            'species': metadata["train"]["selected_class"],}
-        )
+    _df = get_dataset()
 
     if dbg_show_extra:
         # add a few samples to visualise colours 

src/pages/2_🌍_map.py

@@ -6,8 +6,6 @@ st.set_page_config(
     layout="wide",
 )
 
-from utils.st_logs import parse_log_buffer, init_logging_session_states
-
 from maps.obs_map import add_obs_map_header 
 from maps.alps_map import present_alps_map
 from maps.obs_map import present_obs_map
@@ -16,10 +14,6 @@ from datasets import disable_caching
 disable_caching()
 
 ############################################################
-# TO- DO: MAKE ENV FILE 
-# the dataset of observations (hf dataset in our space)
-dataset_id = "Saving-Willy/temp_dataset"
-data_files = "data/train-00000-of-00001.parquet"
 USE_BASIC_MAP = False
 DEV_SIDEBAR_LIB = True
 ############################################################
@@ -35,10 +29,7 @@ with tab_map_ui_cols[1]:
     
 if show_db_points:
     # show a nicer map, observations marked, tileset selectable.
-    st_observation = present_obs_map(
-        dataset_id=dataset_id, data_files=data_files,
-        dbg_show_extra=dbg_show_extra)
-    
+    st_observation = present_obs_map(dbg_show_extra=dbg_show_extra)
 else:
     # development map.
     st_observation = present_alps_map()

src/pages/5_🤝_requests.py

@@ -5,4 +5,13 @@ st.set_page_config(
     page_icon="🤝",
 )
 
-from utils.st_logs import parse_log_buffer, init_logging_session_states
+from utils.st_logs import parse_log_buffer, init_logging_session_states
+
+from datasets import disable_caching
+disable_caching()
+
+############################################################
+# the dataset of observations (hf dataset in our space)
+dataset_id = "Saving-Willy/temp_dataset"
+data_files = "data/train-00000-of-00001.parquet"
+############################################################