diff --git "a/app.py" "b/app.py"
--- "a/app.py"
+++ "b/app.py"
@@ -1,4 +1,2785 @@
-import streamlit as st
-
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+# app.py
+
+import streamlit as st
+
+# Set page config first, before any other st commands
+st.set_page_config(page_title="SNAP", layout="wide")
+
+# Add warning filters
+import warnings
+# More specific warning filters for torch.classes
+warnings.filterwarnings('ignore', message='.*torch.classes.*__path__._path.*')
+warnings.filterwarnings('ignore', message='.*torch.classes.*registered via torch::class_.*')
+
+import pandas as pd
+import numpy as np
+import os
+import io
+import time
+from datetime import datetime
+import base64
+import re
+import pickle
+from typing import List, Dict, Any, Tuple
+import plotly.express as px
+import torch
+
+# For parallelism
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+
+# Import necessary libraries for embeddings, clustering, and summarization
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+from bertopic import BERTopic
+from hdbscan import HDBSCAN
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+# For summarization and chat
+from langchain.chains import LLMChain
+from langchain_community.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate
+from openai import OpenAI
+from transformers import GPT2TokenizerFast
+
+# Initialize OpenAI client and tokenizer
+client = OpenAI()
+
+###############################################################################
+# Helper: Attempt to get this file's directory or fallback to current working dir
+###############################################################################
+def get_base_dir():
+    try:
+        base_dir = os.path.dirname(__file__)
+        if not base_dir:
+            return os.getcwd()
+        return base_dir
+    except NameError:
+        # In case __file__ is not defined (some environments)
+        return os.getcwd()
+
+BASE_DIR = get_base_dir()
+
+# Function to get or create model directory
+def get_model_dir():
+    base_dir = get_base_dir()
+    model_dir = os.path.join(base_dir, 'models')
+    os.makedirs(model_dir, exist_ok=True)
+    return model_dir
+
+# Function to load tokenizer from local storage or download
+def load_tokenizer():
+    model_dir = get_model_dir()
+    tokenizer_dir = os.path.join(model_dir, 'tokenizer')
+    os.makedirs(tokenizer_dir, exist_ok=True)
+    
+    try:
+        # Try to load from local directory first
+        tokenizer = GPT2TokenizerFast.from_pretrained(tokenizer_dir)
+        #st.success("Loaded tokenizer from local storage")
+    except Exception as e:
+        #st.warning("Downloading tokenizer (one-time operation)...")
+        try:
+            # Download and save to local directory
+            tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")  # Use standard GPT2 tokenizer
+            tokenizer.save_pretrained(tokenizer_dir)
+            #st.success("Downloaded and saved tokenizer")
+        except Exception as download_e:
+            #st.error(f"Error downloading tokenizer: {str(download_e)}")
+            raise
+    
+    return tokenizer
+
+# Load tokenizer
+try:
+    tokenizer = load_tokenizer()
+except Exception as e:
+    #st.error("Failed to load tokenizer. Some functionality may be limited.")
+    tokenizer = None
+
+MAX_CONTEXT_WINDOW = 128000  # GPT-4o context window size
+
+# Initialize chat history in session state if not exists
+if 'chat_history' not in st.session_state:
+    st.session_state.chat_history = []
+
+###############################################################################
+# Helper: Get chat response from OpenAI
+###############################################################################
+def get_chat_response(messages):
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=messages,
+            temperature=0,
+        )
+        return response.choices[0].message.content.strip()
+    except Exception as e:
+        st.error(f"Error querying OpenAI: {e}")
+        return None
+
+###############################################################################
+# Helper: Generate raw summary for a cluster (without references)
+###############################################################################
+def generate_raw_cluster_summary(
+    topic_val: int,
+    cluster_df: pd.DataFrame,
+    llm: Any,
+    chat_prompt: Any
+) -> Dict[str, Any]:
+    """Generate a summary for a single cluster without reference enhancement,
+       automatically trimming text if it exceeds a safe token limit."""
+    cluster_text = " ".join(cluster_df['text'].tolist())
+    if not cluster_text.strip():
+        return None
+
+    # Define a safe limit (95% of max context window to leave room for prompts)
+    safe_limit = int(MAX_CONTEXT_WINDOW * 0.95)
+    
+    # Encode the text into tokens
+    encoded_text = tokenizer.encode(cluster_text, add_special_tokens=False)
+    
+    # If the text is too large, slice it
+    if len(encoded_text) > safe_limit:
+        #st.warning(f"Cluster {topic_val} text is too large ({len(encoded_text)} tokens). Trimming to {safe_limit} tokens.")
+        encoded_text = encoded_text[:safe_limit]
+        cluster_text = tokenizer.decode(encoded_text)
+    
+    user_prompt_local = f"**Text to summarize**: {cluster_text}"
+    try:
+        local_chain = LLMChain(llm=llm, prompt=chat_prompt)
+        summary_local = local_chain.run(user_prompt=user_prompt_local).strip()
+        return {'Topic': topic_val, 'Summary': summary_local}
+    except Exception as e:
+        st.error(f"Error generating summary for cluster {topic_val}: {str(e)}")
+        return None
+
+###############################################################################
+# Helper: Enhance a summary with references
+###############################################################################
+def enhance_summary_with_references(
+    summary_dict: Dict[str, Any],
+    df_scope: pd.DataFrame,
+    reference_id_column: str,
+    url_column: str = None,
+    llm: Any = None
+) -> Dict[str, Any]:
+    """Add references to a summary."""
+    if not summary_dict or 'Summary' not in summary_dict:
+        return summary_dict
+    
+    try:
+        cluster_df = df_scope[df_scope['Topic'] == summary_dict['Topic']]
+        enhanced = add_references_to_summary(
+            summary_dict['Summary'],
+            cluster_df,
+            reference_id_column,
+            url_column,
+            llm
+        )
+        summary_dict['Enhanced_Summary'] = enhanced
+        return summary_dict
+    except Exception as e:
+        st.error(f"Error enhancing summary for cluster {summary_dict.get('Topic')}: {str(e)}")
+        return summary_dict
+
+###############################################################################
+# Helper: Process summaries in parallel
+###############################################################################
+def process_summaries_in_parallel(
+    df_scope: pd.DataFrame,
+    unique_selected_topics: List[int],
+    llm: Any,
+    chat_prompt: Any,
+    enable_references: bool = False,
+    reference_id_column: str = None,
+    url_column: str = None,
+    max_workers: int = 16
+) -> List[Dict[str, Any]]:
+    """Process multiple cluster summaries in parallel using ThreadPoolExecutor."""
+    summaries = []
+    total_topics = len(unique_selected_topics)
+    
+    # Create progress placeholders
+    progress_text = st.empty()
+    progress_bar = st.progress(0)
+    
+    try:
+        # Phase 1: Generate raw summaries in parallel
+        progress_text.text(f"Phase 1/3: Generating cluster summaries in parallel (0/{total_topics} completed)")
+        completed_summaries = 0
+        
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit summary generation tasks
+            future_to_topic = {
+                executor.submit(
+                    generate_raw_cluster_summary,
+                    topic_val,
+                    df_scope[df_scope['Topic'] == topic_val],
+                    llm,
+                    chat_prompt
+                ): topic_val
+                for topic_val in unique_selected_topics
+            }
+            
+            # Process completed summary tasks
+            for future in future_to_topic:
+                try:
+                    result = future.result()
+                    if result:
+                        summaries.append(result)
+                    completed_summaries += 1
+                    # Update progress
+                    progress = completed_summaries / total_topics
+                    progress_bar.progress(progress)
+                    progress_text.text(
+                        f"Phase 1/3: Generating cluster summaries in parallel ({completed_summaries}/{total_topics} completed)"
+                    )
+                except Exception as e:
+                    topic_val = future_to_topic[future]
+                    st.error(f"Error in summary generation for cluster {topic_val}: {str(e)}")
+                    completed_summaries += 1
+                    continue
+        
+        # Phase 2: Enhance summaries with references in parallel (if enabled)
+        if enable_references and reference_id_column and summaries:
+            total_to_enhance = len(summaries)
+            completed_enhancements = 0
+            progress_text.text(f"Phase 2/3: Adding references to summaries (0/{total_to_enhance} completed)")
+            progress_bar.progress(0)
+            
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # Submit reference enhancement tasks
+                future_to_summary = {
+                    executor.submit(
+                        enhance_summary_with_references,
+                        summary_dict,
+                        df_scope,
+                        reference_id_column,
+                        url_column,
+                        llm
+                    ): summary_dict.get('Topic')
+                    for summary_dict in summaries
+                }
+                
+                # Process completed enhancement tasks
+                enhanced_summaries = []
+                for future in future_to_summary:
+                    try:
+                        result = future.result()
+                        if result:
+                            enhanced_summaries.append(result)
+                        completed_enhancements += 1
+                        # Update progress
+                        progress = completed_enhancements / total_to_enhance
+                        progress_bar.progress(progress)
+                        progress_text.text(
+                            f"Phase 2/3: Adding references to summaries ({completed_enhancements}/{total_to_enhance} completed)"
+                        )
+                    except Exception as e:
+                        topic_val = future_to_summary[future]
+                        st.error(f"Error in reference enhancement for cluster {topic_val}: {str(e)}")
+                        completed_enhancements += 1
+                        continue
+                
+                summaries = enhanced_summaries
+
+        # Phase 3: Generate cluster names in parallel
+        if summaries:
+            total_to_name = len(summaries)
+            completed_names = 0
+            progress_text.text(f"Phase 3/3: Generating cluster names (0/{total_to_name} completed)")
+            progress_bar.progress(0)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # Submit cluster naming tasks
+                future_to_summary = {
+                    executor.submit(
+                        generate_cluster_name,
+                        summary_dict.get('Enhanced_Summary', summary_dict['Summary']),
+                        llm
+                    ): summary_dict.get('Topic')
+                    for summary_dict in summaries
+                }
+
+                # Process completed naming tasks
+                named_summaries = []
+                for future in future_to_summary:
+                    try:
+                        cluster_name = future.result()
+                        topic_val = future_to_summary[future]
+                        # Find the corresponding summary dict
+                        summary_dict = next(s for s in summaries if s['Topic'] == topic_val)
+                        summary_dict['Cluster_Name'] = cluster_name
+                        named_summaries.append(summary_dict)
+                        completed_names += 1
+                        # Update progress
+                        progress = completed_names / total_to_name
+                        progress_bar.progress(progress)
+                        progress_text.text(
+                            f"Phase 3/3: Generating cluster names ({completed_names}/{total_to_name} completed)"
+                        )
+                    except Exception as e:
+                        topic_val = future_to_summary[future]
+                        st.error(f"Error in cluster naming for cluster {topic_val}: {str(e)}")
+                        completed_names += 1
+                        continue
+
+                summaries = named_summaries
+    finally:
+        # Clean up progress indicators
+        progress_text.empty()
+        progress_bar.empty()
+    
+    return summaries
+
+###############################################################################
+# Helper: Generate cluster name
+###############################################################################
+def generate_cluster_name(summary_text: str, llm: Any) -> str:
+    """Generate a concise, descriptive name for a cluster based on its summary."""
+    system_prompt = """You are a cluster naming expert. Your task is to generate a very concise (3-6 words) but descriptive name for a cluster based on its summary. The name should capture the main theme or focus of the cluster.
+
+Rules:
+1. Keep it between 3-6 words
+2. Be specific but concise
+3. Capture the main theme/focus
+4. Use title case
+4. Do not include words like "Cluster", "Topic", or "Theme"
+5. Focus on the content, not metadata
+
+Example good names:
+- Agricultural Water Management Innovation
+- Gender Equality in Farming
+- Climate-Smart Village Implementation
+- Sustainable Livestock Practices"""
+
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": f"Generate a concise cluster name based on this summary:\n\n{summary_text}"}
+    ]
+
+    try:
+        response = get_chat_response(messages)
+        # Clean up response (remove quotes, newlines, etc.)
+        cluster_name = response.strip().strip('"').strip("'").strip()
+        return cluster_name
+    except Exception as e:
+        st.error(f"Error generating cluster name: {str(e)}")
+        return "Unnamed Cluster"
+
+###############################################################################
+# Helper: Attempt to get this file's directory or fallback to current working dir
+###############################################################################
+def get_base_dir():
+    try:
+        base_dir = os.path.dirname(__file__)
+        if not base_dir:
+            return os.getcwd()
+        return base_dir
+    except NameError:
+        # In case __file__ is not defined (some environments)
+        return os.getcwd()
+
+BASE_DIR = get_base_dir()
+
+###############################################################################
+# NLTK Resource Initialization
+###############################################################################
+def init_nltk_resources():
+    """Initialize NLTK resources with better error handling and less verbose output"""
+    nltk.data.path.append('/home/appuser/nltk_data')  # Ensure consistent data path
+    
+    resources = {
+        'tokenizers/punkt': 'punkt_tab',  # Updated to use punkt_tab
+        'corpora/stopwords': 'stopwords'
+    }
+    
+    for resource_path, resource_name in resources.items():
+        try:
+            nltk.data.find(resource_path)
+        except LookupError:
+            try:
+                nltk.download(resource_name, quiet=True)
+            except Exception as e:
+                st.warning(f"Error downloading NLTK resource {resource_name}: {e}")
+    
+    # Test tokenizer silently
+    try:
+        from nltk.tokenize import PunktSentenceTokenizer
+        tokenizer = PunktSentenceTokenizer()
+        tokenizer.tokenize("Test sentence.")
+    except Exception as e:
+        st.error(f"Error initializing NLTK tokenizer: {e}")
+        try:
+            nltk.download('punkt_tab', quiet=True)  # Updated to use punkt_tab
+        except Exception as e:
+            st.error(f"Failed to download punkt_tab tokenizer: {e}")
+
+# Initialize NLTK resources
+init_nltk_resources()
+
+###############################################################################
+# Function: add_references_to_summary
+###############################################################################
+def add_references_to_summary(summary, source_df, reference_column, url_column=None, llm=None):
+    """
+    Add references to a summary by identifying which parts of the summary come 
+    from which source documents. References will be appended as [ID], 
+    optionally linked if a URL column is provided.
+
+    Args:
+        summary (str): The summary text to enhance with references.
+        source_df (DataFrame): DataFrame containing the source documents.
+        reference_column (str): Column name to use for reference IDs.
+        url_column (str, optional): Column name containing URLs for hyperlinks.
+        llm (LLM, optional): Language model for source attribution.
+    Returns:
+        str: Enhanced summary with references as HTML if possible.
+    """
+    if summary.strip() == "" or source_df.empty or reference_column not in source_df.columns:
+        return summary
+    
+    # If no LLM is provided, we can't do source attribution
+    if llm is None:
+        return summary
+
+    # Split the summary into paragraphs first
+    paragraphs = summary.split('\n\n')
+    enhanced_paragraphs = []
+
+    # Prepare source texts with their reference IDs
+    source_texts = []
+    reference_ids = []
+    urls = []
+    for _, row in source_df.iterrows():
+        if 'text' in row and pd.notna(row['text']) and pd.notna(row[reference_column]):
+            source_texts.append(str(row['text']))
+            reference_ids.append(str(row[reference_column]))
+            if url_column and url_column in row and pd.notna(row[url_column]):
+                urls.append(str(row[url_column]))
+            else:
+                urls.append(None)
+    if not source_texts:
+        return summary
+
+    # Create a mapping between URLs and reference IDs
+    url_map = {}
+    for ref_id, u in zip(reference_ids, urls):
+        if u:
+            url_map[ref_id] = u
+
+    # Define the system prompt for source attribution
+    system_prompt = """
+    You are an expert at identifying the source of information. You will be given:
+    1. A sentence or bullet point from a summary
+    2. A list of source texts with their IDs
+    
+    Your task is to identify which source text(s) the text most likely came from.
+    Return ONLY the IDs of the source texts that contributed to the text, separated by commas.
+    If you cannot confidently attribute the text to any source, return "unknown".
+    """
+
+    for paragraph in paragraphs:
+        if not paragraph.strip():
+            enhanced_paragraphs.append('')
+            continue
+
+        # Check if it's a bullet point list
+        if any(line.strip().startswith('- ') or line.strip().startswith('* ') for line in paragraph.split('\n')):
+            # Handle bullet points
+            bullet_lines = paragraph.split('\n')
+            enhanced_bullets = []
+            for line in bullet_lines:
+                if not line.strip():
+                    enhanced_bullets.append(line)
+                    continue
+                
+                if line.strip().startswith('- ') or line.strip().startswith('* '):
+                    # Process each bullet point
+                    user_prompt = f"""
+                    Text: {line.strip()}
+
+                    Source texts:
+                    {'\n'.join([f"ID: {ref_id}, Text: {text[:500]}..." for ref_id, text in zip(reference_ids, source_texts)])}
+
+                    Which source ID(s) did this text most likely come from? Return only the ID(s) separated by commas, or "unknown".
+                    """
+
+                    try:
+                        system_message = SystemMessagePromptTemplate.from_template(system_prompt)
+                        human_message = HumanMessagePromptTemplate.from_template("{user_prompt}")
+                        chat_prompt = ChatPromptTemplate.from_messages([system_message, human_message])
+                        chain = LLMChain(llm=llm, prompt=chat_prompt)
+                        response = chain.run(user_prompt=user_prompt)
+                        source_ids = response.strip()
+
+                        if source_ids.lower() == "unknown":
+                            enhanced_bullets.append(line)
+                        else:
+                            # Extract just the IDs
+                            source_ids = re.sub(r'[^0-9,\s]', '', source_ids)
+                            source_ids = re.sub(r'\s+', '', source_ids)
+                            ids = [id_.strip() for id_ in source_ids.split(',') if id_.strip()]
+                            
+                            if ids:
+                                ref_parts = []
+                                for id_ in ids:
+                                    if id_ in url_map:
+                                        ref_parts.append(f'<a href="{url_map[id_]}" target="_blank">{id_}</a>')
+                                    else:
+                                        ref_parts.append(id_)
+                                ref_string = ", ".join(ref_parts)
+                                enhanced_bullets.append(f"{line} [{ref_string}]")
+                            else:
+                                enhanced_bullets.append(line)
+                    except Exception:
+                        enhanced_bullets.append(line)
+                else:
+                    enhanced_bullets.append(line)
+            
+            enhanced_paragraphs.append('\n'.join(enhanced_bullets))
+        else:
+            # Handle regular paragraphs
+            sentences = re.split(r'(?<=[.!?])\s+', paragraph)
+            enhanced_sentences = []
+
+            for sentence in sentences:
+                if not sentence.strip():
+                    continue
+
+                user_prompt = f"""
+                Sentence: {sentence.strip()}
+
+                Source texts:
+                {'\n'.join([f"ID: {ref_id}, Text: {text[:500]}..." for ref_id, text in zip(reference_ids, source_texts)])}
+
+                Which source ID(s) did this sentence most likely come from? Return only the ID(s) separated by commas, or "unknown".
+                """
+
+                try:
+                    system_message = SystemMessagePromptTemplate.from_template(system_prompt)
+                    human_message = HumanMessagePromptTemplate.from_template("{user_prompt}")
+                    chat_prompt = ChatPromptTemplate.from_messages([system_message, human_message])
+                    chain = LLMChain(llm=llm, prompt=chat_prompt)
+                    response = chain.run(user_prompt=user_prompt)
+                    source_ids = response.strip()
+
+                    if source_ids.lower() == "unknown":
+                        enhanced_sentences.append(sentence)
+                    else:
+                        # Extract just the IDs
+                        source_ids = re.sub(r'[^0-9,\s]', '', source_ids)
+                        source_ids = re.sub(r'\s+', '', source_ids)
+                        ids = [id_.strip() for id_ in source_ids.split(',') if id_.strip()]
+                        
+                        if ids:
+                            ref_parts = []
+                            for id_ in ids:
+                                if id_ in url_map:
+                                    ref_parts.append(f'<a href="{url_map[id_]}" target="_blank">{id_}</a>')
+                                else:
+                                    ref_parts.append(id_)
+                            ref_string = ", ".join(ref_parts)
+                            enhanced_sentences.append(f"{sentence} [{ref_string}]")
+                        else:
+                            enhanced_sentences.append(sentence)
+                except Exception:
+                    enhanced_sentences.append(sentence)
+
+            enhanced_paragraphs.append(' '.join(enhanced_sentences))
+
+    # Join paragraphs back together with double newlines to preserve formatting
+    return '\n\n'.join(enhanced_paragraphs)
+
+
+st.sidebar.image("static/SNAP_logo.png", width=350)
+
+###############################################################################
+# Device / GPU Info
+###############################################################################
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+if device == 'cuda':
+    st.sidebar.success(f"Using GPU: {torch.cuda.get_device_name(0)}")
+else:
+    st.sidebar.info("Using CPU")
+
+###############################################################################
+# Load or Compute Embeddings
+###############################################################################
+@st.cache_resource
+def get_embedding_model():
+    model_dir = get_model_dir()
+    st_model_dir = os.path.join(model_dir, 'sentence_transformer')
+    os.makedirs(st_model_dir, exist_ok=True)
+    
+    model_name = 'all-MiniLM-L6-v2'
+    try:
+        # Try to load from local directory first
+        model = SentenceTransformer(st_model_dir)
+        #st.success("Loaded sentence transformer from local storage")
+    except Exception as e:
+        #st.warning("Downloading sentence transformer model (one-time operation)...")
+        try:
+            # Download and save to local directory
+            model = SentenceTransformer(model_name)
+            model.save(st_model_dir)
+            #st.success("Downloaded and saved sentence transformer model")
+        except Exception as download_e:
+            st.error(f"Error downloading sentence transformer model: {str(download_e)}")
+            raise
+    
+    return model.to(device)
+
+def generate_embeddings(texts, model):
+    with st.spinner('Calculating embeddings...'):
+        embeddings = model.encode(texts, show_progress_bar=True, device=device)
+    return embeddings
+
+@st.cache_data
+def load_default_dataset(default_dataset_path):
+    if os.path.exists(default_dataset_path):
+        df_ = pd.read_excel(default_dataset_path)
+        return df_
+    else:
+        st.error("Default dataset not found. Please ensure the file exists in the 'input' directory.")
+        return None
+
+@st.cache_data
+def load_uploaded_dataset(uploaded_file):
+    df_ = pd.read_excel(uploaded_file)
+    return df_
+
+def load_or_compute_embeddings(df, using_default_dataset, uploaded_file_name=None, text_columns=None):
+    """
+    Loads pre-computed embeddings from a pickle file if they match current data, 
+    otherwise computes and caches them.
+    """
+    if not text_columns:
+        return None, None
+
+    base_name = "PRMS_2022_2023_2024_QAed" if using_default_dataset else "custom_dataset"
+    if uploaded_file_name:
+        base_name = os.path.splitext(uploaded_file_name)[0]
+
+    cols_key = "_".join(sorted(text_columns))
+    timestamp_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+    embeddings_dir = BASE_DIR
+    if using_default_dataset:
+        embeddings_file = os.path.join(embeddings_dir, f'{base_name}_{cols_key}.pkl')
+    else:
+        # For custom dataset, we still try to avoid regenerating each time
+        embeddings_file = os.path.join(embeddings_dir, f"{base_name}_{cols_key}.pkl")
+
+    df_fill = df.fillna("")
+    texts = df_fill[text_columns].astype(str).agg(' '.join, axis=1).tolist()
+
+    # If already in session_state with matching columns and length, reuse
+    if ('embeddings' in st.session_state 
+        and 'last_text_columns' in st.session_state 
+        and st.session_state['last_text_columns'] == text_columns 
+        and len(st.session_state['embeddings']) == len(texts)):
+        return st.session_state['embeddings'], st.session_state.get('embeddings_file', None)
+
+    # Try to load from disk
+    if os.path.exists(embeddings_file):
+        with open(embeddings_file, 'rb') as f:
+            embeddings = pickle.load(f)
+        if len(embeddings) == len(texts):
+            st.write("Loaded pre-calculated embeddings.")
+            st.session_state['embeddings'] = embeddings
+            st.session_state['embeddings_file'] = embeddings_file
+            st.session_state['last_text_columns'] = text_columns
+            return embeddings, embeddings_file
+
+    # Otherwise compute
+    st.write("Generating embeddings...")
+    model = get_embedding_model()
+    embeddings = generate_embeddings(texts, model)
+    with open(embeddings_file, 'wb') as f:
+        pickle.dump(embeddings, f)
+
+    st.session_state['embeddings'] = embeddings
+    st.session_state['embeddings_file'] = embeddings_file
+    st.session_state['last_text_columns'] = text_columns
+    return embeddings, embeddings_file
+
+
+###############################################################################
+# Reset Filter Function
+###############################################################################
+def reset_filters():
+    st.session_state['selected_additional_filters'] = {}
+
+# Selector de vista
+st.sidebar.radio("Select view", ["Automatic Mode", "Power User Mode"], key="view")
+
+if st.session_state.view == "Power User Mode":      
+    st.header("Power User Mode")     
+    ###############################################################################
+    # Sidebar: Dataset Selection
+    ###############################################################################
+    st.sidebar.title("Data Selection")
+    dataset_option = st.sidebar.selectbox('Select Dataset', ('PRMS 2022+2023+2024 QAed', 'Upload my dataset'))
+
+    if 'df' not in st.session_state:
+        st.session_state['df'] = pd.DataFrame()
+    if 'filtered_df' not in st.session_state:
+        st.session_state['filtered_df'] = pd.DataFrame()
+
+    if dataset_option == 'PRMS 2022+2023+2024 QAed':
+        default_dataset_path = os.path.join(BASE_DIR, 'input', 'export_data_table_results_20251203_101413CET.xlsx')
+        df = load_default_dataset(default_dataset_path)
+        if df is not None:
+            st.session_state['df'] = df.copy()
+            st.session_state['using_default_dataset'] = True
+            
+            # Initialize filtered_df with full dataset by default
+            if 'filtered_df' not in st.session_state or st.session_state['filtered_df'].empty:
+                st.session_state['filtered_df'] = df.copy()
+            
+            # Initialize filter_state if not exists
+            if 'filter_state' not in st.session_state:
+                st.session_state['filter_state'] = {
+                    'applied': False,
+                    'filters': {}
+                }
+            
+            # Set default text columns if not already set
+            if 'text_columns' not in st.session_state or not st.session_state['text_columns']:
+                default_text_cols = []
+                if 'Title' in df.columns and 'Description' in df.columns:
+                    default_text_cols = ['Title', 'Description']
+                st.session_state['text_columns'] = default_text_cols
+            
+            #st.write("Using default dataset:")
+            #st.write("Data Preview:")
+            #st.dataframe(st.session_state['filtered_df'].head(), hide_index=True)
+            #st.write(f"Total number of results: {len(st.session_state['filtered_df'])}")
+
+            df_cols = df.columns.tolist()
+
+            # Additional filter columns
+            st.subheader("Select Filters")
+            if 'additional_filters_selected' not in st.session_state:
+                st.session_state['additional_filters_selected'] = []
+            if 'filter_values' not in st.session_state:
+                st.session_state['filter_values'] = {}
+
+            with st.form("filter_selection_form"):
+                all_columns = df.columns.tolist()
+                selected_additional_cols = st.multiselect(
+                    "Select columns from your dataset to use as filters:",
+                    all_columns,
+                    default=st.session_state['additional_filters_selected']
+                )
+                add_filters_submitted = st.form_submit_button("Add Additional Filters")
+
+            if add_filters_submitted:
+                if selected_additional_cols != st.session_state['additional_filters_selected']:
+                    st.session_state['additional_filters_selected'] = selected_additional_cols
+                    # Reset removed columns
+                    st.session_state['filter_values'] = {
+                        k: v for k, v in st.session_state['filter_values'].items() 
+                        if k in selected_additional_cols
+                    }
+
+            # Show dynamic filters form if any selected columns
+            if st.session_state['additional_filters_selected']:
+                st.subheader("Apply Filters")
+                
+                # Quick search section (outside form)
+                for col_name in st.session_state['additional_filters_selected']:
+                    unique_vals = sorted(df[col_name].dropna().unique().tolist())
+                    
+                    # Add a search box for quick selection
+                    search_key = f"search_{col_name}"
+                    if search_key not in st.session_state:
+                        st.session_state[search_key] = ""
+                        
+                    col1, col2 = st.columns([3, 1])
+                    with col1:
+                        search_term = st.text_input(
+                            f"Search in {col_name}",
+                            key=search_key,
+                            help="Enter text to find and select all matching values"
+                        )
+                    with col2:
+                        if st.button(f"Select Matching", key=f"select_{col_name}"):
+                            # Handle comma-separated values
+                            if search_term:
+                                matching_vals = [
+                                    val for val in unique_vals
+                                    if any(search_term.lower() in str(part).lower() 
+                                        for part in (val.split(',') if isinstance(val, str) else [val]))
+                                ]
+                                # Update the multiselect default value
+                                current_selected = st.session_state['filter_values'].get(col_name, [])
+                                st.session_state['filter_values'][col_name] = list(set(current_selected + matching_vals))
+                                
+                                # Show feedback about matches
+                                if matching_vals:
+                                    st.success(f"Found and selected {len(matching_vals)} matching values")
+                                else:
+                                    st.warning("No matching values found")
+
+                # Filter application form
+                with st.form("apply_filters_form"):
+                    for col_name in st.session_state['additional_filters_selected']:
+                        unique_vals = sorted(df[col_name].dropna().unique().tolist())
+                        selected_vals = st.multiselect(
+                            f"Filter by {col_name}",
+                            options=unique_vals,
+                            default=st.session_state['filter_values'].get(col_name, [])
+                        )
+                        st.session_state['filter_values'][col_name] = selected_vals
+
+                    # Add clear filters button and apply filters button
+                    col1, col2 = st.columns([1, 4])
+                    with col1:
+                        clear_filters = st.form_submit_button("Clear All")
+                    with col2:
+                        apply_filters_submitted = st.form_submit_button("Apply Filters to Dataset")
+
+                    if clear_filters:
+                        st.session_state['filter_values'] = {}
+                        # Clear any existing summary data when filters are cleared
+                        if 'summary_df' in st.session_state:
+                            del st.session_state['summary_df']
+                        if 'high_level_summary' in st.session_state:
+                            del st.session_state['high_level_summary']
+                        if 'enhanced_summary' in st.session_state:
+                            del st.session_state['enhanced_summary']
+                        st.rerun()
+
+            # Text columns selection moved to Advanced Settings
+            with st.expander("⚙️ Advanced Settings", expanded=False):
+                st.subheader("**Select Text Columns for Embedding**")
+                text_columns_selected = st.multiselect(
+                    "Text Columns:",
+                    df_cols,
+                    default=st.session_state['text_columns'],
+                    help="Choose columns containing text for semantic search and clustering. "
+                        "If multiple are selected, their text will be concatenated."
+                )
+                st.session_state['text_columns'] = text_columns_selected
+
+            # Apply filters to the dataset
+            filtered_df = df.copy()
+            if 'apply_filters_submitted' in locals() and apply_filters_submitted:
+                # Clear any existing summary data when new filters are applied
+                if 'summary_df' in st.session_state:
+                    del st.session_state['summary_df']
+                if 'high_level_summary' in st.session_state:
+                    del st.session_state['high_level_summary']
+                if 'enhanced_summary' in st.session_state:
+                    del st.session_state['enhanced_summary']
+                
+                for col_name in st.session_state['additional_filters_selected']:
+                    selected_vals = st.session_state['filter_values'].get(col_name, [])
+                    if selected_vals:
+                        filtered_df = filtered_df[filtered_df[col_name].isin(selected_vals)]
+                st.success("Filters applied successfully!")
+                st.session_state['filtered_df'] = filtered_df.copy()
+                st.session_state['filter_state'] = {
+                    'applied': True,
+                    'filters': st.session_state['filter_values'].copy()
+                }
+                # Reset any existing clustering results
+                for k in ['clustered_data', 'topic_model', 'current_clustering_data',
+                        'current_clustering_option', 'hierarchy']:
+                    if k in st.session_state:
+                        del st.session_state[k]
+
+            elif 'filter_state' in st.session_state and st.session_state['filter_state']['applied']:
+                # Reapply stored filters
+                for col_name, selected_vals in st.session_state['filter_state']['filters'].items():
+                    if selected_vals:
+                        filtered_df = filtered_df[filtered_df[col_name].isin(selected_vals)]
+                st.session_state['filtered_df'] = filtered_df.copy()
+
+            # Show current data preview and download button
+            if st.session_state['filtered_df'] is not None:
+                if st.session_state['filter_state']['applied']:
+                    st.write("Filtered Data Preview:")
+                else:
+                    st.write("Current Data Preview:")
+                st.dataframe(st.session_state['filtered_df'].head(), hide_index=True)
+                st.write(f"Total number of results: {len(st.session_state['filtered_df'])}")
+
+                output = io.BytesIO()
+                writer = pd.ExcelWriter(output, engine='openpyxl')
+                st.session_state['filtered_df'].to_excel(writer, index=False)
+                writer.close()
+                processed_data = output.getvalue()
+
+                st.download_button(
+                    label="Download Current Data",
+                    data=processed_data,
+                    file_name='data.xlsx',
+                    mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+                )
+        else:
+            st.warning("Please ensure the default dataset exists in the 'input' directory.")
+
+    else:
+        # Upload custom dataset
+        uploaded_file = st.sidebar.file_uploader("Upload your Excel file", type=["xlsx"])
+        if uploaded_file is not None:
+            df = load_uploaded_dataset(uploaded_file)
+            if df is not None:
+                st.session_state['df'] = df.copy()
+                st.session_state['using_default_dataset'] = False
+                st.session_state['uploaded_file_name'] = uploaded_file.name
+                st.write("Data preview:")
+                st.write(df.head())
+                df_cols = df.columns.tolist()
+
+                st.subheader("**Select Text Columns for Embedding**")
+                text_columns_selected = st.multiselect(
+                    "Text Columns:", 
+                    df_cols, 
+                    default=df_cols[:1] if df_cols else []
+                )
+                st.session_state['text_columns'] = text_columns_selected
+
+                st.write("**Additional Filters**")
+                selected_additional_cols = st.multiselect(
+                    "Select additional columns from your dataset to use as filters:", 
+                    df_cols, 
+                    default=[]
+                )
+                st.session_state['additional_filters_selected'] = selected_additional_cols
+
+                filtered_df = df.copy()
+                for col_name in selected_additional_cols:
+                    if f'selected_filter_{col_name}' not in st.session_state:
+                        st.session_state[f'selected_filter_{col_name}'] = []
+                    unique_vals = sorted(df[col_name].dropna().unique().tolist())
+                    selected_vals = st.multiselect(
+                        f"Filter by {col_name}",
+                        options=unique_vals,
+                        default=st.session_state[f'selected_filter_{col_name}']
+                    )
+                    st.session_state[f'selected_filter_{col_name}'] = selected_vals
+                    if selected_vals:
+                        filtered_df = filtered_df[filtered_df[col_name].isin(selected_vals)]
+
+                st.session_state['filtered_df'] = filtered_df
+                st.write("Filtered Data Preview:")
+                st.dataframe(filtered_df.head(), hide_index=True)
+                st.write(f"Total number of results: {len(filtered_df)}")
+
+                output = io.BytesIO()
+                writer = pd.ExcelWriter(output, engine='openpyxl')
+                filtered_df.to_excel(writer, index=False)
+                writer.close()
+                processed_data = output.getvalue()
+
+                st.download_button(
+                    label="Download Filtered Data",
+                    data=processed_data,
+                    file_name='filtered_data.xlsx',
+                    mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+                )
+            else:
+                st.warning("Failed to load the uploaded dataset.")
+        else:
+            st.warning("Please upload an Excel file to proceed.")
+
+        if 'filtered_df' in st.session_state:
+            st.write(f"Total number of results: {len(st.session_state['filtered_df'])}")
+
+
+    ###############################################################################
+    # Preserve active tab across reruns
+    ###############################################################################
+    if 'active_tab_index' not in st.session_state:
+        st.session_state.active_tab_index = 0
+
+    tabs_titles = ["Semantic Search", "Clustering", "Summarization", "Chat", "Help"]
+    tabs = st.tabs(tabs_titles)
+    # We just create these references so we can navigate more easily
+    tab_semantic, tab_clustering, tab_summarization, tab_chat, tab_help = tabs
+
+    ###############################################################################
+    # Tab: Help
+    ###############################################################################
+    with tab_help:
+        st.header("Help")
+        st.markdown("""
+        ### About SNAP
+
+        SNAP allows you to explore, filter, search, cluster, and summarize textual datasets.
+
+        **Workflow**:
+        1. **Data Selection (Sidebar)**: Choose the default dataset or upload your own.
+        2. **Filtering**: Set additional filters for your dataset.
+        3. **Select Text Columns**: Which columns to embed.
+        4. **Semantic Search** (Tab): Provide a query and threshold to find relevant documents.
+        5. **Clustering** (Tab): Group documents into topics.
+        6. **Summarization** (Tab): Summarize the clustered documents (with optional references).
+
+        ### Troubleshooting
+        - If you see no results, try lowering the similarity threshold or removing negative/required keywords.
+        - Ensure you have at least one text column selected for embeddings.
+        """)
+
+    ###############################################################################
+    # Tab: Semantic Search
+    ###############################################################################
+    with tab_semantic:
+        st.header("Semantic Search")
+        if 'filtered_df' in st.session_state and not st.session_state['filtered_df'].empty:
+            text_columns = st.session_state.get('text_columns', [])
+            if not text_columns:
+                st.warning("No text columns selected. Please select at least one column for text embedding.")
+            else:
+                df_full = st.session_state['df']
+                # Load or compute embeddings if necessary
+                embeddings, _ = load_or_compute_embeddings(
+                    df_full,
+                    st.session_state.get('using_default_dataset', False),
+                    st.session_state.get('uploaded_file_name'),
+                    text_columns
+                )
+
+                if embeddings is not None:
+                    with st.expander("ℹ️ How Semantic Search Works", expanded=False):
+                        st.markdown("""
+                        ### Understanding Semantic Search
+
+                        Unlike traditional keyword search that looks for exact matches, semantic search understands the meaning and context of your query. Here's how it works:
+
+                        1. **Query Processing**:
+                            - Your search query is converted into a numerical representation (embedding) that captures its meaning
+                            - Example: Searching for "Climate Smart Villages" will understand the concept, not just the words
+                            - Related terms like "sustainable communities", "resilient farming", or "agricultural adaptation" might be found even if they don't contain the exact words
+
+                        2. **Similarity Matching**:
+                            - Documents are ranked by how closely their meaning matches your query
+                            - The similarity threshold controls how strict this matching is
+                            - Higher threshold (e.g., 0.8) = more precise but fewer results
+                            - Lower threshold (e.g., 0.3) = more results but might be less relevant
+
+                        3. **Advanced Features**:
+                            - **Negative Keywords**: Use to explicitly exclude documents containing certain terms
+                            - **Required Keywords**: Ensure specific terms appear in the results
+                            - These work as traditional keyword filters after the semantic search
+
+                        ### Search Tips
+
+                        - **Phrase Queries**: Enter complete phrases for better context
+                            - "Climate Smart Villages" (as one concept)
+                            - Better than separate terms: "climate", "smart", "villages"
+
+                        - **Descriptive Queries**: Add context for better results
+                            - Instead of: "water"
+                            - Better: "water management in agriculture"
+
+                        - **Conceptual Queries**: Focus on concepts rather than specific terms
+                            - Instead of: "increased yield"
+                            - Better: "agricultural productivity improvements"
+
+                        ### Example Searches
+
+                        1. **Query**: "Climate Smart Villages"
+                            - Will find: Documents about climate-resilient communities, adaptive farming practices, sustainable village development
+                            - Even if they don't use these exact words
+
+                        2. **Query**: "Gender equality in agriculture"
+                            - Will find: Women's empowerment in farming, female farmer initiatives, gender-inclusive rural development
+                            - Related concepts are captured semantically
+
+                        3. **Query**: "Sustainable water management"
+                            + Required keyword: "irrigation"
+                            - Combines semantic understanding of water sustainability with specific irrigation focus
+                        """)
+
+                    with st.form("search_parameters"):
+                        query = st.text_input("Enter your search query:")
+                        include_keywords = st.text_input("Include only documents containing these words (comma-separated):")
+                        similarity_threshold = st.slider("Similarity threshold", 0.0, 1.0, 0.35)
+                        submitted = st.form_submit_button("Search")
+
+                    if submitted:
+                        if query.strip():
+                            with st.spinner("Performing Semantic Search..."):
+                                # Clear any existing summary data when new search is run
+                                if 'summary_df' in st.session_state:
+                                    del st.session_state['summary_df']
+                                if 'high_level_summary' in st.session_state:
+                                    del st.session_state['high_level_summary']
+                                if 'enhanced_summary' in st.session_state:
+                                    del st.session_state['enhanced_summary']
+
+                                model = get_embedding_model()
+                                df_filtered = st.session_state['filtered_df'].fillna("")
+                                search_texts = df_filtered[text_columns].agg(' '.join, axis=1).tolist()
+
+                                # Filter the embeddings to the same subset
+                                subset_indices = df_filtered.index
+                                subset_embeddings = embeddings[subset_indices]
+
+                                query_embedding = model.encode([query], device=device)
+                                similarities = cosine_similarity(query_embedding, subset_embeddings)[0]
+
+                                # Show distribution
+                                fig = px.histogram(
+                                    x=similarities,
+                                    nbins=30,
+                                    labels={'x': 'Similarity Score', 'y': 'Number of Documents'},
+                                    title='Distribution of Similarity Scores'
+                                )
+                                fig.add_vline(
+                                    x=similarity_threshold,
+                                    line_dash="dash",
+                                    line_color="red",
+                                    annotation_text=f"Threshold: {similarity_threshold:.2f}",
+                                    annotation_position="top"
+                                )
+                                st.write("### Similarity Score Distribution")
+                                st.plotly_chart(fig)
+
+                                above_threshold_indices = np.where(similarities > similarity_threshold)[0]
+                                if len(above_threshold_indices) == 0:
+                                    st.warning("No results found above the similarity threshold.")
+                                    if 'search_results' in st.session_state:
+                                        del st.session_state['search_results']
+                                else:
+                                    selected_indices = subset_indices[above_threshold_indices]
+                                    results = df_filtered.loc[selected_indices].copy()
+                                    results['similarity_score'] = similarities[above_threshold_indices]
+                                    results.sort_values(by='similarity_score', ascending=False, inplace=True)
+
+                                    # Include keyword filtering
+                                    if include_keywords.strip():
+                                        inc_words = [w.strip().lower() for w in include_keywords.split(',') if w.strip()]
+                                        if inc_words:
+                                            results = results[
+                                                results.apply(
+                                                    lambda row: all(
+                                                        w in (' '.join(row.astype(str)).lower()) for w in inc_words
+                                                    ), 
+                                                    axis=1
+                                                )
+                                            ]
+                                    
+                                    if results.empty:
+                                        st.warning("No results found after applying keyword filters.")
+                                        if 'search_results' in st.session_state:
+                                            del st.session_state['search_results']
+                                    else:
+                                        st.session_state['search_results'] = results.copy()
+                                        output = io.BytesIO()
+                                        writer = pd.ExcelWriter(output, engine='openpyxl')
+                                        results.to_excel(writer, index=False)
+                                        writer.close()
+                                        processed_data = output.getvalue()
+                                        st.session_state['search_results_processed_data'] = processed_data
+                        else:
+                            st.warning("Please enter a query to search.")
+
+                    # Display search results if available
+                    if 'search_results' in st.session_state and not st.session_state['search_results'].empty:
+                        st.write("## Search Results")
+                        results = st.session_state['search_results']
+                        cols_to_display = [c for c in results.columns if c != 'similarity_score'] + ['similarity_score']
+                        st.dataframe(results[cols_to_display], hide_index=True)
+                        st.write(f"Total number of results: {len(results)}")
+
+                        if 'search_results_processed_data' in st.session_state:
+                            st.download_button(
+                                label="Download Full Results",
+                                data=st.session_state['search_results_processed_data'],
+                                file_name='search_results.xlsx',
+                                mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+                                key='download_search_results'
+                            )
+                    else:
+                        st.info("No search results to display. Enter a query and click 'Search'.")
+                else:
+                    st.warning("No embeddings available because no text columns were chosen.")
+        else:
+            st.warning("Filtered dataset is empty or not loaded. Please adjust your filters or upload data.")
+
+
+    ###############################################################################
+    # Tab: Clustering
+    ###############################################################################
+    with tab_clustering:
+        st.header("Clustering")
+        if 'filtered_df' in st.session_state and not st.session_state['filtered_df'].empty:
+            # Add explanation about clustering
+            with st.expander("ℹ️ How Clustering Works", expanded=False):
+                st.markdown("""
+                ### Understanding Document Clustering
+
+                Clustering automatically groups similar documents together, helping you discover patterns and themes in your data. Here's how it works:
+
+                1. **Cluster Formation**:
+                - Documents are grouped based on their semantic similarity
+                - Each cluster represents a distinct theme or topic
+                - Documents that are too different from others may remain unclustered (labeled as -1)
+                - The "Min Cluster Size" parameter controls how clusters are formed
+
+                2. **Interpreting Results**:
+                - Each cluster is assigned a number (e.g., 0, 1, 2...)
+                - Cluster -1 contains "outlier" documents that didn't fit well in other clusters
+                - The size of each cluster indicates how common that theme is
+                - Keywords for each cluster show the main topics/concepts
+
+                3. **Visualizations**:
+                - **Intertopic Distance Map**: Shows how clusters relate to each other
+                    - Closer clusters are more semantically similar
+                    - Size of circles indicates number of documents
+                    - Hover to see top terms for each cluster
+                
+                - **Topic Document Visualization**: Shows individual documents
+                    - Each point is a document
+                    - Colors indicate cluster membership
+                    - Distance between points shows similarity
+                
+                - **Topic Hierarchy**: Shows how topics are related
+                    - Tree structure shows topic relationships
+                    - Parent topics contain broader themes
+                    - Child topics show more specific sub-themes
+
+                ### How to Use Clusters
+
+                1. **Exploration**:
+                - Use clusters to discover main themes in your data
+                - Look for unexpected groupings that might reveal insights
+                - Identify outliers that might need special attention
+
+                2. **Analysis**:
+                - Compare cluster sizes to understand theme distribution
+                - Examine keywords to understand what defines each cluster
+                - Use hierarchy to see how themes are nested
+
+                3. **Practical Applications**:
+                - Generate summaries for specific clusters
+                - Focus detailed analysis on clusters of interest
+                - Use clusters to organize and categorize documents
+                - Identify gaps or overlaps in your dataset
+
+                ### Tips for Better Results
+
+                - **Adjust Min Cluster Size**:
+                - Larger values (15-20): Fewer, broader clusters
+                - Smaller values (2-5): More specific, smaller clusters
+                - Balance between too many small clusters and too few large ones
+
+                - **Choose Data Wisely**:
+                - Cluster full dataset for overall themes
+                - Cluster search results for focused analysis
+                - More documents generally give better clusters
+
+                - **Interpret with Context**:
+                - Consider your domain knowledge
+                - Look for patterns across multiple visualizations
+                - Use cluster insights to guide further analysis
+                """)
+
+            df_to_cluster = None
+            
+            # Create a single form for clustering settings
+            with st.form("clustering_form"):
+                st.subheader("Clustering Settings")
+                
+                # Data source selection
+                clustering_option = st.radio(
+                    "Select data for clustering:",
+                    ('Full Dataset', 'Filtered Dataset', 'Semantic Search Results')
+                )
+                
+                # Clustering parameters
+                min_cluster_size_val = st.slider(
+                    "Min Cluster Size",
+                    min_value=2,
+                    max_value=50,
+                    value=st.session_state.get('min_cluster_size', 5),
+                    help="Minimum size of each cluster in HDBSCAN; In other words, it's the minimum number of documents/texts that must be grouped together to form a valid cluster.\n\n- A larger value (e.g., 20) will result in fewer, larger clusters\n- A smaller value (e.g., 2-5) will allow for more clusters, including smaller ones\n- Documents that don't fit into any cluster meeting this minimum size requirement are labeled as noise (typically assigned to cluster -1)"
+                )
+                
+                run_clustering = st.form_submit_button("Run Clustering")
+                
+                if run_clustering:
+                    st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                    st.session_state['min_cluster_size'] = min_cluster_size_val
+
+            # Decide which DataFrame is used based on the selection
+            if clustering_option == 'Semantic Search Results':
+                if 'search_results' in st.session_state and not st.session_state['search_results'].empty:
+                    df_to_cluster = st.session_state['search_results'].copy()
+                else:
+                    st.warning("No semantic search results found. Please run a search first.")
+            elif clustering_option == 'Filtered Dataset':
+                if 'filtered_df' in st.session_state and not st.session_state['filtered_df'].empty:
+                    df_to_cluster = st.session_state['filtered_df'].copy()
+                else:
+                    st.warning("Filtered dataset is empty. Please check your filters.")
+            else:  # Full Dataset
+                if 'df' in st.session_state and not st.session_state['df'].empty:
+                    df_to_cluster = st.session_state['df'].copy()
+
+            text_columns = st.session_state.get('text_columns', [])
+            if not text_columns:
+                st.warning("No text columns selected. Please select text columns to embed before clustering.")
+            else:
+                # Ensure embeddings are available
+                df_full = st.session_state['df']
+                embeddings, _ = load_or_compute_embeddings(
+                    df_full,
+                    st.session_state.get('using_default_dataset', False),
+                    st.session_state.get('uploaded_file_name'),
+                    text_columns
+                )
+
+                if df_to_cluster is not None and embeddings is not None and not df_to_cluster.empty and run_clustering:
+                    with st.spinner("Performing clustering..."):
+                        # Clear any existing summary data when clustering is run
+                        if 'summary_df' in st.session_state:
+                            del st.session_state['summary_df']
+                        if 'high_level_summary' in st.session_state:
+                            del st.session_state['high_level_summary']
+                        if 'enhanced_summary' in st.session_state:
+                            del st.session_state['enhanced_summary']
+
+                        dfc = df_to_cluster.copy().fillna("")
+                        dfc['text'] = dfc[text_columns].astype(str).agg(' '.join, axis=1)
+
+                        # Filter embeddings to those rows
+                        selected_indices = dfc.index
+                        embeddings_clustering = embeddings[selected_indices]
+
+                        # Basic cleaning
+                        stop_words = set(stopwords.words('english'))
+                        texts_cleaned = []
+                        for text in dfc['text'].tolist():
+                            try:
+                                # First try with word_tokenize
+                                try:
+                                    word_tokens = word_tokenize(text)
+                                except LookupError:
+                                    # If punkt is missing, try downloading it again
+                                    nltk.download('punkt_tab', quiet=False)
+                                    word_tokens = word_tokenize(text)
+                                except Exception as e:
+                                    # If word_tokenize fails, fall back to simple splitting
+                                    st.warning(f"Using fallback tokenization due to error: {e}")
+                                    word_tokens = text.split()
+                                
+                                filtered_text = ' '.join([w for w in word_tokens if w.lower() not in stop_words])
+                                texts_cleaned.append(filtered_text)
+                            except Exception as e:
+                                st.error(f"Error processing text: {e}")
+                                # Add the original text if processing fails
+                                texts_cleaned.append(text)
+
+                        try:
+                            # Validation checks before clustering
+                            if len(texts_cleaned) < min_cluster_size_val:
+                                st.error(f"Not enough documents to form clusters. You have {len(texts_cleaned)} documents but minimum cluster size is set to {min_cluster_size_val}.")
+                                st.session_state['clustering_error'] = "Insufficient documents for clustering"
+                                st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                                st.stop()
+
+                            # Convert embeddings to CPU numpy if needed
+                            if torch.is_tensor(embeddings_clustering):
+                                embeddings_for_clustering = embeddings_clustering.cpu().numpy()
+                            else:
+                                embeddings_for_clustering = embeddings_clustering
+
+                            # Additional validation
+                            if embeddings_for_clustering.shape[0] != len(texts_cleaned):
+                                st.error("Mismatch between number of embeddings and texts.")
+                                st.session_state['clustering_error'] = "Embedding and text count mismatch"
+                                st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                                st.stop()
+
+                            # Build the HDBSCAN model with error handling
+                            try:
+                                hdbscan_model = HDBSCAN(
+                                    min_cluster_size=min_cluster_size_val, 
+                                    metric='euclidean', 
+                                    cluster_selection_method='eom'
+                                )
+                                
+                                # Build the BERTopic model
+                                topic_model = BERTopic(
+                                    embedding_model=get_embedding_model(),
+                                    hdbscan_model=hdbscan_model
+                                )
+
+                                # Fit the model and get topics
+                                topics, probs = topic_model.fit_transform(
+                                    texts_cleaned, 
+                                    embeddings=embeddings_for_clustering
+                                )
+
+                                # Validate clustering results
+                                unique_topics = set(topics)
+                                if len(unique_topics) < 2:
+                                    st.warning("Clustering resulted in too few clusters. Retry or try reducing the minimum cluster size.")
+                                    if -1 in unique_topics:
+                                        non_noise_docs = sum(1 for t in topics if t != -1)
+                                        st.info(f"Only {non_noise_docs} documents were assigned to clusters. The rest were marked as noise (-1).")
+                                        if non_noise_docs < min_cluster_size_val:
+                                            st.error("Not enough documents were successfully clustered. Try reducing the minimum cluster size.")
+                                            st.session_state['clustering_error'] = "Insufficient clustered documents"
+                                            st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                                            st.stop()
+
+                                # Store results if validation passes
+                                dfc['Topic'] = topics
+                                st.session_state['topic_model'] = topic_model
+                                st.session_state['clustered_data'] = dfc.copy()
+                                st.session_state['clustering_texts_cleaned'] = texts_cleaned
+                                st.session_state['clustering_embeddings'] = embeddings_for_clustering
+                                st.session_state['clustering_completed'] = True
+
+                                # Try to generate visualizations with error handling
+                                try:
+                                    st.session_state['intertopic_distance_fig'] = topic_model.visualize_topics()
+                                except Exception as viz_error:
+                                    st.warning("Could not generate topic visualization. This usually happens when there are too few total clusters. Try adjusting the minimum cluster size or adding more documents.")
+                                    st.session_state['intertopic_distance_fig'] = None
+
+                                try:
+                                    st.session_state['topic_document_fig'] = topic_model.visualize_documents(
+                                        texts_cleaned, 
+                                        embeddings=embeddings_for_clustering
+                                    )
+                                except Exception as viz_error:
+                                    st.warning("Could not generate document visualization. This might happen when the clustering results are not optimal. Try adjusting the clustering parameters.")
+                                    st.session_state['topic_document_fig'] = None
+
+                                try:
+                                    hierarchy = topic_model.hierarchical_topics(texts_cleaned)
+                                    st.session_state['hierarchy'] = hierarchy if hierarchy is not None else pd.DataFrame()
+                                    st.session_state['hierarchy_fig'] = topic_model.visualize_hierarchy()
+                                except Exception as viz_error:
+                                    st.warning("Could not generate topic hierarchy visualization. This usually happens when there aren't enough distinct topics to form a hierarchy.")
+                                    st.session_state['hierarchy'] = pd.DataFrame()
+                                    st.session_state['hierarchy_fig'] = None
+
+                            except ValueError as ve:
+                                if "zero-size array to reduction operation maximum which has no identity" in str(ve):
+                                    st.error("Clustering failed: No valid clusters could be formed. Try reducing the minimum cluster size.")
+                                elif "Cannot use scipy.linalg.eigh for sparse A with k > N" in str(ve):
+                                    st.error("Clustering failed: Too many components requested for the number of documents. Try with more documents or adjust clustering parameters.")
+                                else:
+                                    st.error(f"Clustering error: {str(ve)}")
+                                st.session_state['clustering_error'] = str(ve)
+                                st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                                st.stop()
+
+                        except Exception as e:
+                            st.error(f"An error occurred during clustering: {str(e)}")
+                            st.session_state['clustering_error'] = str(e)
+                            st.session_state['clustering_completed'] = False
+                            st.session_state.active_tab_index = tabs_titles.index("Clustering")
+                            st.stop()
+
+                    # Display clustering results if they exist
+                    if st.session_state.get('clustering_completed', False):
+                        st.subheader("Topic Overview")
+                        dfc = st.session_state['clustered_data']
+                        topic_model = st.session_state['topic_model']
+                        topics = dfc['Topic'].tolist()
+                        
+                        unique_topics = sorted(list(set(topics)))
+                        cluster_info = []
+                        for t in unique_topics:
+                            cluster_docs = dfc[dfc['Topic'] == t]
+                            count = len(cluster_docs)
+                            top_words = topic_model.get_topic(t)
+                            if top_words:
+                                top_keywords = ", ".join([w[0] for w in top_words[:5]])
+                            else:
+                                top_keywords = "N/A"
+                            cluster_info.append((t, count, top_keywords))
+                        cluster_df = pd.DataFrame(cluster_info, columns=["Topic", "Count", "Top Keywords"])
+
+                        st.write("### Topic Overview")
+                        st.dataframe(
+                            cluster_df,
+                            column_config={
+                                "Topic": st.column_config.NumberColumn("Topic", help="Topic ID (-1 represents outliers)"),
+                                "Count": st.column_config.NumberColumn("Count", help="Number of documents in this topic"),
+                                "Top Keywords": st.column_config.TextColumn(
+                                    "Top Keywords",
+                                    help="Top 5 keywords that characterize this topic"
+                                )
+                            },
+                            hide_index=True
+                        )
+
+                        st.subheader("Clustering Results")
+                        columns_to_display = [c for c in dfc.columns if c != 'text']
+                        st.dataframe(dfc[columns_to_display], hide_index=True)
+
+                        # Display stored visualizations with error handling
+                        st.write("### Intertopic Distance Map")
+                        if st.session_state.get('intertopic_distance_fig') is not None:
+                            try:
+                                st.plotly_chart(st.session_state['intertopic_distance_fig'])
+                            except Exception:
+                                st.info("Topic visualization is not available for the current clustering results.")
+
+                        st.write("### Topic Document Visualization")
+                        if st.session_state.get('topic_document_fig') is not None:
+                            try:
+                                st.plotly_chart(st.session_state['topic_document_fig'])
+                            except Exception:
+                                st.info("Document visualization is not available for the current clustering results.")
+
+                        st.write("### Topic Hierarchy")
+                        if st.session_state.get('hierarchy_fig') is not None:
+                            try:
+                                st.plotly_chart(st.session_state['hierarchy_fig'])
+                            except Exception:
+                                st.info("Topic hierarchy visualization is not available for the current clustering results.")
+                if not (df_to_cluster is not None and embeddings is not None and not df_to_cluster.empty and run_clustering):
+                    pass
+        else:
+            st.warning("Please select or upload a dataset and filter as needed.")
+
+
+    ###############################################################################
+    # Tab: Summarization
+    ###############################################################################
+    with tab_summarization:
+        st.header("Summarization")
+        # Add explanation about summarization
+        with st.expander("ℹ️ How Summarization Works", expanded=False):
+            st.markdown("""
+            ### Understanding Document Summarization
+
+            Summarization condenses multiple documents into concise, meaningful summaries while preserving key information. Here's how it works:
+
+            1. **Summary Generation**:
+            - Documents are processed using advanced language models
+            - Key themes and important points are identified
+            - Content is condensed while maintaining context
+            - Both high-level and cluster-specific summaries are available
+
+            2. **Reference System**:
+            - Summaries can include references to source documents
+            - References are shown as [ID] or as clickable links
+            - Each statement can be traced back to its source
+            - Helps maintain accountability and verification
+
+            3. **Types of Summaries**:
+            - **High-Level Summary**: Overview of all selected documents
+                - Captures main themes across the entire selection
+                - Ideal for quick understanding of large document sets
+                - Shows relationships between different topics
+            
+            - **Cluster-Specific Summaries**: Focused on each cluster
+                - More detailed for specific themes
+                - Shows unique aspects of each cluster
+                - Helps understand sub-topics in depth
+
+            ### How to Use Summaries
+
+            1. **Configuration**:
+            - Choose between all clusters or specific ones
+            - Set temperature for creativity vs. consistency
+            - Adjust max tokens for summary length
+            - Enable/disable reference system
+
+            2. **Reference Options**:
+            - Select column for reference IDs
+            - Add hyperlinks to references
+            - Choose URL column for clickable links
+            - References help track information sources
+
+            3. **Practical Applications**:
+            - Quick overview of large datasets
+            - Detailed analysis of specific themes
+            - Evidence-based reporting with references
+            - Compare different document groups
+
+            ### Tips for Better Results
+
+            - **Temperature Setting**:
+            - Higher (0.7-1.0): More creative, varied summaries
+            - Lower (0.1-0.3): More consistent, conservative summaries
+            - Balance based on your needs for creativity vs. consistency
+
+            - **Token Length**:
+            - Longer limits: More detailed summaries
+            - Shorter limits: More concise, focused summaries
+            - Adjust based on document complexity
+
+            - **Reference Usage**:
+            - Enable references for traceability
+            - Use hyperlinks for easy navigation
+            - Choose meaningful reference columns
+            - Helps validate summary accuracy
+
+            ### Best Practices
+
+            1. **For General Overview**:
+            - Use high-level summary
+            - Keep temperature moderate (0.5-0.7)
+            - Enable references for verification
+            - Focus on broader themes
+
+            2. **For Detailed Analysis**:
+            - Use cluster-specific summaries
+            - Adjust temperature based on need
+            - Include references with hyperlinks
+            - Look for patterns within clusters
+
+            3. **For Reporting**:
+            - Combine both summary types
+            - Use references extensively
+            - Balance detail and brevity
+            - Ensure source traceability
+            """)
+
+        df_summ = None
+        # We'll try to summarize either the clustered data or just the filtered dataset
+        if 'clustered_data' in st.session_state and not st.session_state['clustered_data'].empty:
+            df_summ = st.session_state['clustered_data']
+        elif 'filtered_df' in st.session_state and not st.session_state['filtered_df'].empty:
+            df_summ = st.session_state['filtered_df']
+        else:
+            st.warning("No data available for summarization. Please cluster first or have some filtered data.")
+        
+        if df_summ is not None and not df_summ.empty:
+            text_columns = st.session_state.get('text_columns', [])
+            if not text_columns:
+                st.warning("No text columns selected. Please select columns for text embedding first.")
+            else:
+                if 'Topic' not in df_summ.columns or 'topic_model' not in st.session_state:
+                    st.warning("No 'Topic' column found. Summaries per cluster are only available if you've run clustering.")
+                else:
+                    topic_model = st.session_state['topic_model']
+                    df_summ['text'] = df_summ.fillna("").astype(str)[text_columns].agg(' '.join, axis=1)
+
+                    # List of topics
+                    topics = sorted(df_summ['Topic'].unique())
+                    cluster_info = []
+                    for t in topics:
+                        cluster_docs = df_summ[df_summ['Topic'] == t]
+                        count = len(cluster_docs)
+                        top_words = topic_model.get_topic(t)
+                        if top_words:
+                            top_keywords = ", ".join([w[0] for w in top_words[:5]])
+                        else:
+                            top_keywords = "N/A"
+                        cluster_info.append((t, count, top_keywords))
+                    cluster_df = pd.DataFrame(cluster_info, columns=["Topic", "Count", "Top Keywords"])
+
+                    # If we have cluster names from previous summarization, add them
+                    if 'summary_df' in st.session_state and 'Cluster_Name' in st.session_state['summary_df'].columns:
+                        summary_df = st.session_state['summary_df']
+                        # Create a mapping of topic to name for merging
+                        topic_names = {t: name for t, name in zip(summary_df['Topic'], summary_df['Cluster_Name'])}
+                        # Add cluster names to cluster_df
+                        cluster_df['Cluster_Name'] = cluster_df['Topic'].map(lambda x: topic_names.get(x, 'Unnamed Cluster'))
+                        # Reorder columns to show name after topic
+                        cluster_df = cluster_df[['Topic', 'Cluster_Name', 'Count', 'Top Keywords']]
+                        
+                    st.write("### Available Clusters:")
+                    st.dataframe(
+                        cluster_df,
+                        column_config={
+                            "Topic": st.column_config.NumberColumn("Topic", help="Topic ID (-1 represents outliers)"),
+                            "Cluster_Name": st.column_config.TextColumn("Cluster Name", help="AI-generated name describing the cluster theme"),
+                            "Count": st.column_config.NumberColumn("Count", help="Number of documents in this topic"),
+                            "Top Keywords": st.column_config.TextColumn(
+                                "Top Keywords",
+                                help="Top 5 keywords that characterize this topic"
+                            )
+                        },
+                        hide_index=True
+                    )
+
+                    # Summarization settings
+                    st.subheader("Summarization Settings")
+                    # Summaries scope
+                    summary_scope = st.radio(
+                        "Generate summaries for:",
+                        ["All clusters", "Specific clusters"]
+                    )
+                    if summary_scope == "Specific clusters":
+                        # Format options to include cluster names if available
+                        if 'Cluster_Name' in cluster_df.columns:
+                            topic_options = [f"Cluster {t} - {name}" for t, name in zip(cluster_df['Topic'], cluster_df['Cluster_Name'])]
+                            topic_to_id = {opt: t for opt, t in zip(topic_options, cluster_df['Topic'])}
+                            selected_topic_options = st.multiselect("Select clusters to summarize", topic_options)
+                            selected_topics = [topic_to_id[opt] for opt in selected_topic_options]
+                        else:
+                            selected_topics = st.multiselect("Select clusters to summarize", topics)
+                    else:
+                        selected_topics = topics
+
+                    # Add system prompt configuration
+                    default_system_prompt = """You are an expert summarizer skilled in creating concise and relevant summaries.
+    You will be given text and an objective context. Please produce a clear, cohesive,
+    and thematically relevant summary. 
+    Focus on key points, insights, or patterns that emerge from the text."""
+
+                    if 'system_prompt' not in st.session_state:
+                        st.session_state['system_prompt'] = default_system_prompt
+
+                    with st.expander("🔧 Advanced Settings", expanded=False):
+                        st.markdown("""
+                        ### System Prompt Configuration
+                        
+                        The system prompt guides the AI in how to generate summaries. You can customize it to better suit your needs:
+                        - Be specific about the style and focus you want
+                        - Add domain-specific context if needed
+                        - Include any special formatting requirements
+                        """)
+                        
+                        system_prompt = st.text_area(
+                            "Customize System Prompt",
+                            value=st.session_state['system_prompt'],
+                            height=150,
+                            help="This prompt guides the AI in how to generate summaries. Edit it to customize the summary style and focus."
+                        )
+                        
+                        if st.button("Reset to Default"):
+                            system_prompt = default_system_prompt
+                            st.session_state['system_prompt'] = default_system_prompt
+
+                        st.markdown("### Generation Parameters")
+                        temperature = st.slider(
+                            "Temperature",
+                            0.0, 1.0, 0.7,
+                            help="Higher values (0.7-1.0) make summaries more creative but less predictable. Lower values (0.1-0.3) make them more focused and consistent."
+                        )
+                        max_tokens = st.slider(
+                            "Max Tokens",
+                            100, 3000, 1000,
+                            help="Maximum length of generated summaries. Higher values allow for more detailed summaries but take longer to generate."
+                        )
+
+                    st.session_state['system_prompt'] = system_prompt
+
+                    st.write("### Enhanced Summary References")
+                    st.write("Select columns for references (optional).")
+                    all_cols = [c for c in df_summ.columns if c not in ['text', 'Topic', 'similarity_score']]
+                    
+                    # By default, let's guess the first column as reference ID if available
+                    if 'reference_id_column' not in st.session_state:
+                        st.session_state.reference_id_column = all_cols[0] if all_cols else None
+                    # If there's a column that looks like a URL, guess that
+                    url_guess = next((c for c in all_cols if 'url' in c.lower() or 'link' in c.lower()), None)
+                    if 'url_column' not in st.session_state:
+                        st.session_state.url_column = url_guess
+
+                    enable_references = st.checkbox(
+                        "Enable references in summaries", 
+                        value=True,  # default to True as requested
+                        help="Add source references to the final summary text."
+                    )
+                    reference_id_column = st.selectbox(
+                        "Select column to use as reference ID:",
+                        all_cols,
+                        index=all_cols.index(st.session_state.reference_id_column) if st.session_state.reference_id_column in all_cols else 0
+                    )
+                    add_hyperlinks = st.checkbox(
+                        "Add hyperlinks to references",
+                        value=True,  # default to True
+                        help="If the reference column has a matching URL, make it clickable."
+                    )
+                    url_column = None
+                    if add_hyperlinks:
+                        url_column = st.selectbox(
+                            "Select column containing URLs:",
+                            all_cols,
+                            index=all_cols.index(st.session_state.url_column) if (st.session_state.url_column in all_cols) else 0
+                        )
+
+                    # Summarization button
+                    if st.button("Generate Summaries"):
+                        openai_api_key = os.environ.get('OPENAI_API_KEY')
+                        if not openai_api_key:
+                            st.error("OpenAI API key not found. Please set the OPENAI_API_KEY environment variable.")
+                        else:
+                            # Set flag to indicate summarization button was clicked
+                            st.session_state['_summarization_button_clicked'] = True
+                            
+                            llm = ChatOpenAI(
+                                api_key=openai_api_key, 
+                                model_name='gpt-4o-mini',  # or 'gpt-4o'
+                                temperature=temperature, 
+                                max_tokens=max_tokens
+                            )
+
+                            # Filter to selected topics
+                            if selected_topics:
+                                df_scope = df_summ[df_summ['Topic'].isin(selected_topics)]
+                            else:
+                                st.warning("No topics selected for summarization.")
+                                df_scope = pd.DataFrame()
+
+                            if df_scope.empty:
+                                st.warning("No documents match the selected topics for summarization.")
+                            else:
+                                all_texts = df_scope['text'].tolist()
+                                combined_text = " ".join(all_texts)
+                                if not combined_text.strip():
+                                    st.warning("No text data available for summarization.")
+                                else:
+                                    # For cluster-specific summaries, use the customized prompt
+                                    local_system_message = SystemMessagePromptTemplate.from_template(st.session_state['system_prompt'])
+                                    local_human_message = HumanMessagePromptTemplate.from_template("{user_prompt}")
+                                    local_chat_prompt = ChatPromptTemplate.from_messages([local_system_message, local_human_message])
+
+                                    # Summaries per cluster
+                                    # Only if multiple clusters are selected
+                                    unique_selected_topics = df_scope['Topic'].unique()
+                                    if len(unique_selected_topics) > 1:
+                                        st.write("### Summaries per Selected Cluster")
+                                        
+                                        # Process summaries in parallel
+                                        with st.spinner("Generating cluster summaries in parallel..."):
+                                            summaries = process_summaries_in_parallel(
+                                                df_scope=df_scope,
+                                                unique_selected_topics=unique_selected_topics,
+                                                llm=llm,
+                                                chat_prompt=local_chat_prompt,
+                                                enable_references=enable_references,
+                                                reference_id_column=reference_id_column,
+                                                url_column=url_column if add_hyperlinks else None,
+                                                max_workers=min(16, len(unique_selected_topics))  # Limit workers based on clusters
+                                            )
+
+                                    if summaries:
+                                        summary_df = pd.DataFrame(summaries)
+                                        # Store the summaries DataFrame in session state
+                                        st.session_state['summary_df'] = summary_df
+                                        # Store additional summary info in session state
+                                        st.session_state['has_references'] = enable_references
+                                        st.session_state['reference_id_column'] = reference_id_column
+                                        st.session_state['url_column'] = url_column if add_hyperlinks else None
+                                        
+                                        # Update cluster_df with new names
+                                        if 'Cluster_Name' in summary_df.columns:
+                                            topic_names = {t: name for t, name in zip(summary_df['Topic'], summary_df['Cluster_Name'])}
+                                            cluster_df['Cluster_Name'] = cluster_df['Topic'].map(lambda x: topic_names.get(x, 'Unnamed Cluster'))
+                                            cluster_df = cluster_df[['Topic', 'Cluster_Name', 'Count', 'Top Keywords']]
+                                            
+                                            # Immediately display updated cluster overview
+                                            st.write("### Updated Topic Overview:")
+                                            st.dataframe(
+                                                cluster_df,
+                                                column_config={
+                                                    "Topic": st.column_config.NumberColumn("Topic", help="Topic ID (-1 represents outliers)"),
+                                                    "Cluster_Name": st.column_config.TextColumn("Cluster Name", help="AI-generated name describing the cluster theme"),
+                                                    "Count": st.column_config.NumberColumn("Count", help="Number of documents in this topic"),
+                                                    "Top Keywords": st.column_config.TextColumn(
+                                                        "Top Keywords",
+                                                        help="Top 5 keywords that characterize this topic"
+                                                    )
+                                                },
+                                                hide_index=True
+                                            )
+                                            
+                                        # Now generate high-level summary from the cluster summaries
+                                        with st.spinner("Generating high-level summary from cluster summaries..."):
+                                            # Format cluster summaries with proper markdown and HTML
+                                            formatted_summaries = []
+                                            total_tokens = 0
+                                            MAX_SAFE_TOKENS = int(MAX_CONTEXT_WINDOW * 0.75)  # Leave room for system prompt and completion
+                                            summary_batches = []
+                                            current_batch = []
+                                            current_batch_tokens = 0
+
+                                            for _, row in summary_df.iterrows():
+                                                summary_text = row.get('Enhanced_Summary', row['Summary'])
+                                                formatted_summary = f"### Cluster {row['Topic']} Summary:\n\n{summary_text}"
+                                                summary_tokens = len(tokenizer(formatted_summary)["input_ids"])
+                                                
+                                                # If adding this summary would exceed the safe token limit, start a new batch
+                                                if current_batch_tokens + summary_tokens > MAX_SAFE_TOKENS:
+                                                    if current_batch:  # Only append if we have summaries in the current batch
+                                                        summary_batches.append(current_batch)
+                                                        current_batch = []
+                                                        current_batch_tokens = 0
+                                                
+                                                current_batch.append(formatted_summary)
+                                                current_batch_tokens += summary_tokens
+
+                                            # Add the last batch if it has any summaries
+                                            if current_batch:
+                                                summary_batches.append(current_batch)
+
+                                            # Generate overview for each batch
+                                            batch_overviews = []
+                                            with st.spinner("Generating batch summaries..."):
+                                                for i, batch in enumerate(summary_batches, 1):
+                                                    st.write(f"Processing batch {i} of {len(summary_batches)}...")
+                                                    
+                                                    batch_text = "\n\n".join(batch)
+                                                    batch_prompt = f"""Below are summaries from a subset of clusters from results made using Transformers NLP on a set of results from the CGIAR reporting system. Each summary contains references to source documents in the form of hyperlinked IDs like [ID] or <a href="...">ID</a>.
+
+Please create a comprehensive overview that synthesizes these clusters so that both the main themes and findings are covered in an organized way. IMPORTANT: 
+1. Preserve all hyperlinked references exactly as they appear in the input summaries
+2. Maintain the HTML anchor tags (<a href="...">) intact when using information from the summaries
+3. Keep the markdown formatting for better readability
+4. Note that this is part {i} of {len(summary_batches)} parts, so focus on the themes present in these specific clusters
+
+Here are the cluster summaries to synthesize:
+
+{batch_text}"""
+
+                                                    # Generate overview for this batch
+                                                    high_level_system_message = SystemMessagePromptTemplate.from_template(st.session_state['system_prompt'])
+                                                    high_level_human_message = HumanMessagePromptTemplate.from_template("{user_prompt}")
+                                                    high_level_chat_prompt = ChatPromptTemplate.from_messages([high_level_system_message, high_level_human_message])
+                                                    high_level_chain = LLMChain(llm=llm, prompt=high_level_chat_prompt)
+                                                    batch_overview = high_level_chain.run(user_prompt=batch_prompt).strip()
+                                                    batch_overviews.append(batch_overview)
+
+                                            # Now combine the batch overviews
+                                            with st.spinner("Generating final combined summary..."):
+                                                combined_overviews = "\n\n### Part ".join([f"{i+1}:\n\n{overview}" for i, overview in enumerate(batch_overviews)])
+                                                final_prompt = f"""Below are {len(batch_overviews)} overview summaries, each covering different clusters of research results. Each part maintains its original references to source documents.
+
+Please create a final comprehensive synthesis that:
+1. Integrates the key themes and findings from all parts
+2. Preserves all hyperlinked references exactly as they appear
+3. Maintains the HTML anchor tags (<a href="...">) intact
+4. Keeps the markdown formatting for better readability
+5. Creates a coherent narrative across all parts
+6. Highlights any themes that span multiple parts
+
+Here are the overviews to synthesize:
+
+### Part 1:
+
+{combined_overviews}"""
+
+                                                # Verify the final prompt's token count
+                                                final_prompt_tokens = len(tokenizer(final_prompt)["input_ids"])
+                                                if final_prompt_tokens > MAX_SAFE_TOKENS:
+                                                    st.error(f"❌ Final synthesis prompt ({final_prompt_tokens:,} tokens) exceeds safe limit ({MAX_SAFE_TOKENS:,}). Using batch summaries separately.")
+                                                    high_level_summary = "# Overall Summary\n\n" + "\n\n".join([f"## Batch {i+1}\n\n{overview}" for i, overview in enumerate(batch_overviews)])
+                                                else:
+                                                    # Generate final synthesis
+                                                    high_level_chain = LLMChain(llm=llm, prompt=high_level_chat_prompt)
+                                                    high_level_summary = high_level_chain.run(user_prompt=final_prompt).strip()
+
+                                            # Store both versions of the summary
+                                            st.session_state['high_level_summary'] = high_level_summary
+                                            st.session_state['enhanced_summary'] = high_level_summary
+
+                                            # Set flag to indicate summarization is complete
+                                            st.session_state['summarization_completed'] = True
+
+                                            # Update the display without rerunning
+                                            st.write("### High-Level Summary:")
+                                            st.markdown(high_level_summary, unsafe_allow_html=True)
+
+                                        # Display cluster summaries
+                                        st.write("### Cluster Summaries:")
+                                        if enable_references and 'Enhanced_Summary' in summary_df.columns:
+                                            for idx, row in summary_df.iterrows():
+                                                cluster_name = row.get('Cluster_Name', 'Unnamed Cluster')
+                                                st.write(f"**Topic {row['Topic']} - {cluster_name}**")
+                                                st.markdown(row['Enhanced_Summary'], unsafe_allow_html=True)
+                                                st.write("---")
+                                            with st.expander("View original summaries in table format"):
+                                                display_df = summary_df[['Topic', 'Cluster_Name', 'Summary']]
+                                                display_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                                                st.dataframe(display_df, hide_index=True)
+                                        else:
+                                            st.write("### Summaries per Cluster:")
+                                            if 'Cluster_Name' in summary_df.columns:
+                                                display_df = summary_df[['Topic', 'Cluster_Name', 'Summary']]
+                                                display_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                                                st.dataframe(display_df, hide_index=True)
+                                            else:
+                                                st.dataframe(summary_df, hide_index=True)
+
+                                        # Download
+                                        if 'Enhanced_Summary' in summary_df.columns:
+                                            dl_df = summary_df[['Topic', 'Cluster_Name', 'Summary']]
+                                            dl_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                                        else:
+                                            dl_df = summary_df
+                                        csv_bytes = dl_df.to_csv(index=False).encode('utf-8')
+                                        b64 = base64.b64encode(csv_bytes).decode()
+                                        href = f'<a href="data:file/csv;base64,{b64}" download="summaries.csv">Download Summaries CSV</a>'
+                                        st.markdown(href, unsafe_allow_html=True)
+
+            # Display existing summaries if available and summarization was completed
+            if st.session_state.get('summarization_completed', False):
+                if 'summary_df' in st.session_state and not st.session_state['summary_df'].empty:
+                    if 'high_level_summary' in st.session_state:
+                        st.write("### High-Level Summary:")
+                        st.markdown(st.session_state['enhanced_summary'] if st.session_state.get('enhanced_summary') else st.session_state['high_level_summary'], unsafe_allow_html=True)
+
+                    st.write("### Cluster Summaries:")
+                    summary_df = st.session_state['summary_df']
+                    if 'Enhanced_Summary' in summary_df.columns:
+                        for idx, row in summary_df.iterrows():
+                            cluster_name = row.get('Cluster_Name', 'Unnamed Cluster')
+                            st.write(f"**Topic {row['Topic']} - {cluster_name}**")
+                            st.markdown(row['Enhanced_Summary'], unsafe_allow_html=True)
+                            st.write("---")
+                        with st.expander("View original summaries in table format"):
+                            display_df = summary_df[['Topic', 'Cluster_Name', 'Summary']]
+                            display_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                            st.dataframe(display_df, hide_index=True)
+                    else:
+                        st.dataframe(summary_df, hide_index=True)
+
+                    # Add download button for existing summaries
+                    dl_df = summary_df[['Topic', 'Cluster_Name', 'Summary']] if 'Cluster_Name' in summary_df.columns else summary_df
+                    if 'Cluster_Name' in dl_df.columns:
+                        dl_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                    csv_bytes = dl_df.to_csv(index=False).encode('utf-8')
+                    b64 = base64.b64encode(csv_bytes).decode()
+                    href = f'<a href="data:file/csv;base64,{b64}" download="summaries.csv">Download Summaries CSV</a>'
+                    st.markdown(href, unsafe_allow_html=True)
+        else:
+            st.warning("No data available for summarization.")
+
+        # Display existing summaries if available (when returning to the tab)
+        if not st.session_state.get('_summarization_button_clicked', False):  # Only show if not just generated
+            if 'high_level_summary' in st.session_state:
+                st.write("### Existing High-Level Summary:")
+                if st.session_state.get('enhanced_summary'):
+                    st.markdown(st.session_state['enhanced_summary'], unsafe_allow_html=True)
+                    with st.expander("View original summary (without references)"):
+                        st.write(st.session_state['high_level_summary'])
+                else:
+                    st.write(st.session_state['high_level_summary'])
+
+            if 'summary_df' in st.session_state and not st.session_state['summary_df'].empty:
+                st.write("### Existing Cluster Summaries:")
+                summary_df = st.session_state['summary_df']
+                if 'Enhanced_Summary' in summary_df.columns:
+                    for idx, row in summary_df.iterrows():
+                        cluster_name = row.get('Cluster_Name', 'Unnamed Cluster')
+                        st.write(f"**Topic {row['Topic']} - {cluster_name}**")
+                        st.markdown(row['Enhanced_Summary'], unsafe_allow_html=True)
+                        st.write("---")
+                    with st.expander("View original summaries in table format"):
+                        display_df = summary_df[['Topic', 'Cluster_Name', 'Summary']]
+                        display_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                        st.dataframe(display_df, hide_index=True)
+                else:
+                    st.dataframe(summary_df, hide_index=True)
+
+                # Add download button for existing summaries
+                dl_df = summary_df[['Topic', 'Cluster_Name', 'Summary']] if 'Cluster_Name' in summary_df.columns else summary_df
+                if 'Cluster_Name' in dl_df.columns:
+                    dl_df.columns = ['Topic', 'Cluster Name', 'Summary']
+                csv_bytes = dl_df.to_csv(index=False).encode('utf-8')
+                b64 = base64.b64encode(csv_bytes).decode()
+                href = f'<a href="data:file/csv;base64,{b64}" download="summaries.csv">Download Summaries CSV</a>'
+                st.markdown(href, unsafe_allow_html=True)
+
+
+    ###############################################################################
+    # Tab: Chat
+    ###############################################################################
+    with tab_chat:
+        st.header("Chat with Your Data")
+        
+        # Add explanation about chat functionality
+        with st.expander("ℹ️ How Chat Works", expanded=False):
+            st.markdown("""
+            ### Understanding Chat with Your Data
+
+            The chat functionality allows you to have an interactive conversation about your data, whether it's filtered, clustered, or raw. Here's how it works:
+
+            1. **Data Selection**:
+            - Choose which dataset to chat about (filtered, clustered, or search results)
+            - Optionally focus on specific clusters if clustering was performed
+            - System automatically includes relevant context from your selection
+
+            2. **Context Window**:
+            - Shows how much of the GPT-4 context window is being used
+            - Helps you understand if you need to filter data further
+            - Displays token usage statistics
+
+            3. **Chat Features**:
+            - Ask questions about your data
+            - Get insights and analysis
+            - Reference specific documents or clusters
+            - Download chat context for transparency
+
+            ### Best Practices
+
+            1. **Data Selection**:
+            - Start with filtered or clustered data for more focused conversations
+            - Select specific clusters if you want to dive deep into a topic
+            - Consider the context window usage when selecting data
+
+            2. **Asking Questions**:
+            - Be specific in your questions
+            - Ask about patterns, trends, or insights
+            - Reference clusters or documents by their IDs
+            - Build on previous questions for deeper analysis
+
+            3. **Managing Context**:
+            - Monitor the context window usage
+            - Filter data further if context is too full
+            - Download chat context for documentation
+            - Clear chat history to start fresh
+
+            ### Tips for Better Results
+
+            - **Question Types**:
+            - "What are the main themes in cluster 3?"
+            - "Compare the findings between clusters 1 and 2"
+            - "Summarize the methodology used across these documents"
+            - "What are the common outcomes reported?"
+
+            - **Follow-up Questions**:
+            - Build on previous answers
+            - Ask for clarification
+            - Request specific examples
+            - Explore relationships between findings
+            """)
+
+        # Function to check data source availability
+        def get_available_data_sources():
+            sources = []
+            if 'filtered_df' in st.session_state and not st.session_state['filtered_df'].empty:
+                sources.append("Filtered Dataset")
+            if 'clustered_data' in st.session_state and not st.session_state['clustered_data'].empty:
+                sources.append("Clustered Data")
+            if 'search_results' in st.session_state and not st.session_state['search_results'].empty:
+                sources.append("Search Results")
+            if ('high_level_summary' in st.session_state or 
+                ('summary_df' in st.session_state and not st.session_state['summary_df'].empty)):
+                sources.append("Summarized Data")
+            return sources
+
+        # Get available data sources
+        available_sources = get_available_data_sources()
+        
+        if not available_sources:
+            st.warning("No data available for chat. Please filter, cluster, search, or summarize first.")
+            st.stop()
+
+        # Initialize or update data source in session state
+        if 'chat_data_source' not in st.session_state:
+            st.session_state.chat_data_source = available_sources[0]
+        elif st.session_state.chat_data_source not in available_sources:
+            st.session_state.chat_data_source = available_sources[0]
+
+        # Data source selection with automatic fallback
+        data_source = st.radio(
+            "Select data to chat about:",
+            available_sources,
+            index=available_sources.index(st.session_state.chat_data_source),
+            help="Choose which dataset you want to analyze in the chat."
+        )
+
+        # Update session state if data source changed
+        if data_source != st.session_state.chat_data_source:
+            st.session_state.chat_data_source = data_source
+            # Clear any cluster-specific selections if switching data sources
+            if 'chat_selected_cluster' in st.session_state:
+                del st.session_state.chat_selected_cluster
+
+        # Get the appropriate DataFrame based on selected source
+        df_chat = None
+        if data_source == "Filtered Dataset":
+            df_chat = st.session_state['filtered_df']
+        elif data_source == "Clustered Data":
+            df_chat = st.session_state['clustered_data']
+        elif data_source == "Search Results":
+            df_chat = st.session_state['search_results']
+        elif data_source == "Summarized Data":
+            # Create DataFrame with selected summaries
+            summary_rows = []
+            
+            # Add high-level summary if available
+            if 'high_level_summary' in st.session_state:
+                summary_rows.append({
+                    'Summary_Type': 'High-Level Summary',
+                    'Content': st.session_state.get('enhanced_summary', st.session_state['high_level_summary'])
+                })
+            
+            # Add cluster summaries if available
+            if 'summary_df' in st.session_state and not st.session_state['summary_df'].empty:
+                summary_df = st.session_state['summary_df']
+                for _, row in summary_df.iterrows():
+                    summary_rows.append({
+                        'Summary_Type': f"Cluster {row['Topic']} Summary",
+                        'Content': row.get('Enhanced_Summary', row['Summary'])
+                    })
+            
+            if summary_rows:
+                df_chat = pd.DataFrame(summary_rows)
+
+        if df_chat is not None and not df_chat.empty:
+            # If we have clustered data, allow cluster selection
+            selected_cluster = None
+            if data_source != "Summarized Data" and 'Topic' in df_chat.columns:
+                cluster_option = st.radio(
+                    "Choose cluster scope:",
+                    ["All Clusters", "Specific Cluster"]
+                )
+                if cluster_option == "Specific Cluster":
+                    unique_topics = sorted(df_chat['Topic'].unique())
+                    # Check if we have cluster names
+                    if 'summary_df' in st.session_state and 'Cluster_Name' in st.session_state['summary_df'].columns:
+                        summary_df = st.session_state['summary_df']
+                        # Create a mapping of topic to name
+                        topic_names = {t: name for t, name in zip(summary_df['Topic'], summary_df['Cluster_Name'])}
+                        # Format the selectbox options
+                        topic_options = [
+                            (t, f"Cluster {t} - {topic_names.get(t, 'Unnamed Cluster')}")
+                            for t in unique_topics
+                        ]
+                        selected_cluster = st.selectbox(
+                            "Select cluster to focus on:",
+                            [t[0] for t in topic_options],
+                            format_func=lambda x: next(opt[1] for opt in topic_options if opt[0] == x)
+                        )
+                    else:
+                        selected_cluster = st.selectbox(
+                            "Select cluster to focus on:",
+                            unique_topics,
+                            format_func=lambda x: f"Cluster {x}"
+                        )
+                    if selected_cluster is not None:
+                        df_chat = df_chat[df_chat['Topic'] == selected_cluster]
+                        st.session_state.chat_selected_cluster = selected_cluster
+                elif 'chat_selected_cluster' in st.session_state:
+                    del st.session_state.chat_selected_cluster
+
+            # Prepare the data for chat context
+            text_columns = st.session_state.get('text_columns', [])
+            if not text_columns and data_source != "Summarized Data":
+                st.warning("No text columns selected. Please select text columns to enable chat functionality.")
+                st.stop()
+
+            # Instead of limiting to 210 documents, we'll limit by tokens
+            MAX_ALLOWED_TOKENS = int(MAX_CONTEXT_WINDOW * 0.95)  # 95% of context window
+            
+            # Prepare system message first to account for its tokens
+            system_msg = {
+                "role": "system",
+                "content": """You are a specialized assistant analyzing data from a research database. 
+    Your role is to:
+    1. Provide clear, concise answers based on the data provided
+    2. Highlight relevant information from specific results when answering
+    3. When referencing specific results, use their row index or ID if available
+    4. Clearly state if information is not available in the results
+    5. Maintain a professional and analytical tone
+    6. Format your responses using Markdown:
+    - Use **bold** for emphasis
+    - Use bullet points and numbered lists for structured information
+    - Create tables using Markdown syntax when presenting structured data
+    - Use backticks for code or technical terms
+    - Include hyperlinks when referencing external sources
+    - Use headings (###) to organize long responses
+
+    The data is provided in a structured format where:""" + ("""
+    - Each result contains multiple fields
+    - Text content is primarily in the following columns: """ + ", ".join(text_columns) + """
+    - Additional metadata and fields are available for reference
+    - If clusters are present, they are numbered (e.g., Cluster 0, Cluster 1, etc.)""" if data_source != "Summarized Data" else """
+    - The data consists of AI-generated summaries of the documents
+    - Each summary may contain references to source documents in markdown format
+    - References are shown as [ID] or as clickable hyperlinks
+    - Summaries may be high-level (covering all documents) or cluster-specific""") + """
+    """
+            }
+
+            # Calculate system message tokens
+            system_tokens = len(tokenizer(system_msg["content"])["input_ids"])
+            remaining_tokens = MAX_ALLOWED_TOKENS - system_tokens
+
+            # Prepare the data context with token limiting
+            data_text = "Available Data:\n"
+            included_rows = 0
+            total_rows = len(df_chat)
+
+            if data_source == "Summarized Data":
+                # For summarized data, process row by row
+                for idx, row in df_chat.iterrows():
+                    row_text = f"\n{row['Summary_Type']}:\n{row['Content']}\n"
+                    row_tokens = len(tokenizer(row_text)["input_ids"])
+                    
+                    if remaining_tokens - row_tokens > 0:
+                        data_text += row_text
+                        remaining_tokens -= row_tokens
+                        included_rows += 1
+                    else:
+                        break
+            else:
+                # For regular data, process row by row
+                for idx, row in df_chat.iterrows():
+                    row_text = f"\nItem {idx}:\n"
+                    for col in df_chat.columns:
+                        if not pd.isna(row[col]) and str(row[col]).strip() and col != 'similarity_score':
+                            row_text += f"{col}: {row[col]}\n"
+                    
+                    row_tokens = len(tokenizer(row_text)["input_ids"])
+                    if remaining_tokens - row_tokens > 0:
+                        data_text += row_text
+                        remaining_tokens -= row_tokens
+                        included_rows += 1
+                    else:
+                        break
+
+            # Calculate token usage
+            data_tokens = len(tokenizer(data_text)["input_ids"])
+            total_tokens = system_tokens + data_tokens
+            context_usage_percent = (total_tokens / MAX_CONTEXT_WINDOW) * 100
+
+            # Display token usage and data coverage
+            st.subheader("Context Window Usage")
+            st.write(f"System Message: {system_tokens:,} tokens")
+            st.write(f"Data Context: {data_tokens:,} tokens")
+            st.write(f"Total: {total_tokens:,} tokens ({context_usage_percent:.1f}% of available context)")
+            st.write(f"Documents included: {included_rows:,} out of {total_rows:,} ({(included_rows/total_rows*100):.1f}%)")
+            
+            if context_usage_percent > 90:
+                st.warning("⚠️ High context usage! Consider reducing the number of results or filtering further.")
+            elif context_usage_percent > 75:
+                st.info("ℹ️ Moderate context usage. Still room for your question, but consider reducing results if asking a long question.")
+
+            # Add download button for chat context
+            chat_context = f"""System Message:
+    {system_msg['content']}
+
+    {data_text}"""
+            st.download_button(
+                label="📥 Download Chat Context",
+                data=chat_context,
+                file_name="chat_context.txt",
+                mime="text/plain",
+                help="Download the exact context that the chatbot receives"
+            )
+
+            # Chat interface
+            col_chat1, col_chat2 = st.columns([3, 1])
+            with col_chat1:
+                user_input = st.text_area("Ask a question about your data:", key="chat_input")
+            with col_chat2:
+                if st.button("Clear Chat History"):
+                    st.session_state.chat_history = []
+                    st.rerun()
+
+            # Store current tab index before processing
+            current_tab = tabs_titles.index("Chat")
+            
+            if st.button("Send", key="send_button"):
+                if user_input:
+                    # Set the active tab index to stay on Chat
+                    st.session_state.active_tab_index = current_tab
+                    
+                    with st.spinner("Processing your question..."):
+                        # Add user's question to chat history
+                        st.session_state.chat_history.append({"role": "user", "content": user_input})
+                        
+                        # Prepare messages for API call
+                        messages = [system_msg]
+                        messages.append({"role": "user", "content": f"Here is the data to reference:\n\n{data_text}\n\nUser question: {user_input}"})
+                        
+                        # Get response from OpenAI
+                        response = get_chat_response(messages)
+                        
+                        if response:
+                            st.session_state.chat_history.append({"role": "assistant", "content": response})
+
+            # Display chat history
+            st.subheader("Chat History")
+            for message in st.session_state.chat_history:
+                if message["role"] == "user":
+                    st.write("**You:**", message["content"])
+                else:
+                    st.write("**Assistant:**")
+                    st.markdown(message["content"], unsafe_allow_html=True)
+                st.write("---")  # Add a separator between messages
+
+
+    ###############################################################################
+    # Tab: Internal Validation
+    ###############################################################################
+
+else:  # Simple view
+    st.header("Automatic Mode")
+    
+    # Initialize session state for automatic view
+    if 'df' not in st.session_state:
+        default_dataset_path = os.path.join(BASE_DIR, 'input', 'export_data_table_results_20251203_101413CET.xlsx')
+        df = load_default_dataset(default_dataset_path)
+        if df is not None:
+            st.session_state['df'] = df.copy()
+            st.session_state['using_default_dataset'] = True
+            st.session_state['filtered_df'] = df.copy()
+            
+            # Set default text columns if not already set
+            if 'text_columns' not in st.session_state or not st.session_state['text_columns']:
+                default_text_cols = []
+                if 'Title' in df.columns and 'Description' in df.columns:
+                    default_text_cols = ['Title', 'Description']
+                st.session_state['text_columns'] = default_text_cols
+
+    # Single search bar for automatic processing
+    #st.write("Enter your query to automatically search, cluster, and summarize the results:")
+    query = st.text_input("Write your query here:")
+
+    
+
+    
+    if st.button("SNAP!"):
+        if query.strip():
+            # Step 1: Semantic Search
+            st.write("### Step 1: Semantic Search")
+            with st.spinner("Performing Semantic Search..."):
+                text_columns = st.session_state.get('text_columns', [])
+                if text_columns:
+                    df_full = st.session_state['df']
+                    embeddings, _ = load_or_compute_embeddings(
+                        df_full,
+                        st.session_state.get('using_default_dataset', False),
+                        st.session_state.get('uploaded_file_name'),
+                        text_columns
+                    )
+                    
+                    if embeddings is not None:
+                        model = get_embedding_model()
+                        df_filtered = st.session_state['filtered_df'].fillna("")
+                        search_texts = df_filtered[text_columns].agg(' '.join, axis=1).tolist()
+                        
+                        subset_indices = df_filtered.index
+                        subset_embeddings = embeddings[subset_indices]
+                        
+                        query_embedding = model.encode([query], device=device)
+                        similarities = cosine_similarity(query_embedding, subset_embeddings)[0]
+                        
+                        similarity_threshold = 0.35  # Default threshold
+                        above_threshold_indices = np.where(similarities > similarity_threshold)[0]
+                        
+                        if len(above_threshold_indices) > 0:
+                            selected_indices = subset_indices[above_threshold_indices]
+                            results = df_filtered.loc[selected_indices].copy()
+                            results['similarity_score'] = similarities[above_threshold_indices]
+                            results.sort_values(by='similarity_score', ascending=False, inplace=True)
+                            st.session_state['search_results'] = results.copy()
+                            st.write(f"Found {len(results)} relevant documents")
+                        else:
+                            st.warning("No results found above the similarity threshold.")
+                            st.stop()
+                
+            # Step 2: Clustering
+            if 'search_results' in st.session_state and not st.session_state['search_results'].empty:
+                st.write("### Step 2: Clustering")
+                with st.spinner("Performing clustering..."):
+                    df_to_cluster = st.session_state['search_results'].copy()
+                    dfc = df_to_cluster.copy().fillna("")
+                    dfc['text'] = dfc[text_columns].astype(str).agg(' '.join, axis=1)
+                    
+                    # Filter embeddings to those rows
+                    selected_indices = dfc.index
+                    embeddings_clustering = embeddings[selected_indices]
+                    
+                    # Basic cleaning
+                    stop_words = set(stopwords.words('english'))
+                    texts_cleaned = []
+                    for text in dfc['text'].tolist():
+                        try:
+                            word_tokens = word_tokenize(text)
+                            filtered_text = ' '.join([w for w in word_tokens if w.lower() not in stop_words])
+                            texts_cleaned.append(filtered_text)
+                        except Exception as e:
+                            texts_cleaned.append(text)
+                    
+                    min_cluster_size = 5  # Default value
+                    
+                    try:
+                        # Convert embeddings to CPU numpy if needed
+                        if torch.is_tensor(embeddings_clustering):
+                            embeddings_for_clustering = embeddings_clustering.cpu().numpy()
+                        else:
+                            embeddings_for_clustering = embeddings_clustering
+                        
+                        # Build the HDBSCAN model
+                        hdbscan_model = HDBSCAN(
+                            min_cluster_size=min_cluster_size, 
+                            metric='euclidean', 
+                            cluster_selection_method='eom'
+                        )
+                        
+                        # Build the BERTopic model
+                        topic_model = BERTopic(
+                            embedding_model=get_embedding_model(),
+                            hdbscan_model=hdbscan_model
+                        )
+                        
+                        # Fit the model and get topics
+                        topics, probs = topic_model.fit_transform(
+                            texts_cleaned, 
+                            embeddings=embeddings_for_clustering
+                        )
+                        
+                        # Store results
+                        dfc['Topic'] = topics
+                        st.session_state['topic_model'] = topic_model
+                        st.session_state['clustered_data'] = dfc.copy()
+                        st.session_state['clustering_completed'] = True
+
+                        # Display clustering results summary
+                        unique_topics = sorted(list(set(topics)))
+                        num_clusters = len([t for t in unique_topics if t != -1])  # Exclude noise cluster (-1)
+                        noise_docs = len([t for t in topics if t == -1])
+                        clustered_docs = len(topics) - noise_docs
+                        
+                        st.write(f"Found {num_clusters} distinct clusters")
+                        #st.write(f"Documents successfully clustered: {clustered_docs}")
+                        #if noise_docs > 0:
+                        #    st.write(f"Documents not fitting in any cluster: {noise_docs}")
+                        
+                        # Show quick cluster overview
+                        cluster_info = []
+                        for t in unique_topics:
+                            if t != -1:  # Skip noise cluster in the overview
+                                cluster_docs = dfc[dfc['Topic'] == t]
+                                count = len(cluster_docs)
+                                top_words = topic_model.get_topic(t)
+                                top_keywords = ", ".join([w[0] for w in top_words[:5]]) if top_words else "N/A"
+                                cluster_info.append((t, count, top_keywords))
+                        
+                        if cluster_info:
+                            #st.write("### Quick Cluster Overview:")
+                            cluster_df = pd.DataFrame(cluster_info, columns=["Topic", "Count", "Top Keywords"])
+                        #     st.dataframe(
+                        #         cluster_df,
+                        #         column_config={
+                        #             "Topic": st.column_config.NumberColumn("Topic", help="Topic ID"),
+                        #             "Count": st.column_config.NumberColumn("Count", help="Number of documents in this topic"),
+                        #             "Top Keywords": st.column_config.TextColumn(
+                        #                 "Top Keywords",
+                        #                 help="Top 5 keywords that characterize this topic"
+                        #             )
+                        #         },
+                        #         hide_index=True
+                        #     )
+                        
+                        # Generate visualizations
+                        try:
+                            st.session_state['intertopic_distance_fig'] = topic_model.visualize_topics()
+                        except Exception:
+                            st.session_state['intertopic_distance_fig'] = None
+                        
+                        try:
+                            st.session_state['topic_document_fig'] = topic_model.visualize_documents(
+                                texts_cleaned, 
+                                embeddings=embeddings_for_clustering
+                            )
+                        except Exception:
+                            st.session_state['topic_document_fig'] = None
+                        
+                        try:
+                            hierarchy = topic_model.hierarchical_topics(texts_cleaned)
+                            st.session_state['hierarchy'] = hierarchy if hierarchy is not None else pd.DataFrame()
+                            st.session_state['hierarchy_fig'] = topic_model.visualize_hierarchy()
+                        except Exception:
+                            st.session_state['hierarchy'] = pd.DataFrame()
+                            st.session_state['hierarchy_fig'] = None
+                            
+                    except Exception as e:
+                        st.error(f"An error occurred during clustering: {str(e)}")
+                        st.stop()
+                
+                # Step 3: Summarization
+                if st.session_state.get('clustering_completed', False):
+                    st.write("### Step 3: Summarization")
+                    
+                    # Initialize OpenAI client
+                    openai_api_key = os.environ.get('OPENAI_API_KEY')
+                    if not openai_api_key:
+                        st.error("OpenAI API key not found. Please set the OPENAI_API_KEY environment variable.")
+                        st.stop()
+                    
+                    llm = ChatOpenAI(
+                        api_key=openai_api_key, 
+                        model_name='gpt-4o-mini',
+                        temperature=0.7,
+                        max_tokens=1000
+                    )
+                    
+                    df_scope = st.session_state['clustered_data']
+                    unique_selected_topics = df_scope['Topic'].unique()
+                    
+                    # Process summaries in parallel
+                    with st.spinner("Generating summaries..."):
+                        local_system_message = SystemMessagePromptTemplate.from_template("""You are an expert summarizer skilled in creating concise and relevant summaries.
+You will be given text and an objective context. Please produce a clear, cohesive,
+and thematically relevant summary. 
+Focus on key points, insights, or patterns that emerge from the text.""")
+                        local_human_message = HumanMessagePromptTemplate.from_template("{user_prompt}")
+                        local_chat_prompt = ChatPromptTemplate.from_messages([local_system_message, local_human_message])
+                        
+                        # Find URL column if it exists
+                        url_column = next((col for col in df_scope.columns if 'url' in col.lower() or 'link' in col.lower() or 'pdf' in col.lower()), None)
+                        
+                        summaries = process_summaries_in_parallel(
+                            df_scope=df_scope,
+                            unique_selected_topics=unique_selected_topics,
+                            llm=llm,
+                            chat_prompt=local_chat_prompt,
+                            enable_references=True,
+                            reference_id_column=df_scope.columns[0],
+                            url_column=url_column,  # Add URL column for clickable links
+                            max_workers=min(16, len(unique_selected_topics))
+                        )
+                        
+                        if summaries:
+                            summary_df = pd.DataFrame(summaries)
+                            st.session_state['summary_df'] = summary_df
+                            
+                            # Display updated cluster overview
+                            if 'Cluster_Name' in summary_df.columns:
+                                st.write("### Updated Topic Overview:")
+                                cluster_info = []
+                                for t in unique_selected_topics:
+                                    cluster_docs = df_scope[df_scope['Topic'] == t]
+                                    count = len(cluster_docs)
+                                    top_words = topic_model.get_topic(t)
+                                    top_keywords = ", ".join([w[0] for w in top_words[:5]]) if top_words else "N/A"
+                                    cluster_name = summary_df[summary_df['Topic'] == t]['Cluster_Name'].iloc[0]
+                                    cluster_info.append((t, cluster_name, count, top_keywords))
+                                
+                                cluster_df = pd.DataFrame(cluster_info, columns=["Topic", "Cluster_Name", "Count", "Top Keywords"])
+                                st.dataframe(
+                                    cluster_df,
+                                    column_config={
+                                        "Topic": st.column_config.NumberColumn("Topic", help="Topic ID (-1 represents outliers)"),
+                                        "Cluster_Name": st.column_config.TextColumn("Cluster Name", help="AI-generated name describing the cluster theme"),
+                                        "Count": st.column_config.NumberColumn("Count", help="Number of documents in this topic"),
+                                        "Top Keywords": st.column_config.TextColumn(
+                                            "Top Keywords",
+                                            help="Top 5 keywords that characterize this topic"
+                                        )
+                                    },
+                                    hide_index=True
+                                )
+                            
+                            # Generate and display high-level summary
+                            with st.spinner("Generating high-level summary..."):
+                                formatted_summaries = []
+                                summary_batches = []
+                                current_batch = []
+                                current_batch_tokens = 0
+                                MAX_SAFE_TOKENS = int(MAX_CONTEXT_WINDOW * 0.75)
+                                
+                                for _, row in summary_df.iterrows():
+                                    summary_text = row.get('Enhanced_Summary', row['Summary'])
+                                    formatted_summary = f"### Cluster {row['Topic']} Summary:\n\n{summary_text}"
+                                    summary_tokens = len(tokenizer(formatted_summary)["input_ids"])
+                                    
+                                    if current_batch_tokens + summary_tokens > MAX_SAFE_TOKENS:
+                                        if current_batch:
+                                            summary_batches.append(current_batch)
+                                            current_batch = []
+                                            current_batch_tokens = 0
+                                    
+                                    current_batch.append(formatted_summary)
+                                    current_batch_tokens += summary_tokens
+                                
+                                if current_batch:
+                                    summary_batches.append(current_batch)
+                                
+                                # Process each batch separately first
+                                batch_overviews = []
+                                for i, batch in enumerate(summary_batches, 1):
+                                    st.write(f"Processing summary batch {i} of {len(summary_batches)}...")
+                                    batch_text = "\n\n".join(batch)
+                                    batch_prompt = f"""Below are summaries from a subset of clusters from results made using Transformers NLP on a set of results from the CGIAR reporting system. Each summary contains references to source documents in the form of hyperlinked IDs like [ID] or <a href="...">ID</a>.
+
+Please create a comprehensive overview that synthesizes these clusters so that both the main themes and findings are covered in an organized way. IMPORTANT: 
+1. Preserve all hyperlinked references exactly as they appear in the input summaries
+2. Maintain the HTML anchor tags (<a href="...">) intact when using information from the summaries
+3. Keep the markdown formatting for better readability
+4. Create clear sections with headings for different themes
+5. Use bullet points or numbered lists where appropriate
+6. Focus on synthesizing the main themes and findings
+
+Here are the cluster summaries to synthesize:
+
+{batch_text}"""
+                                    
+                                    high_level_chain = LLMChain(llm=llm, prompt=local_chat_prompt)
+                                    batch_overview = high_level_chain.run(user_prompt=batch_prompt).strip()
+                                    batch_overviews.append(batch_overview)
+                                
+                                # Now create the final synthesis
+                                if len(batch_overviews) > 1:
+                                    st.write("Generating final synthesis...")
+                                    combined_overviews = "\n\n# Part ".join([f"{i+1}\n\n{overview}" for i, overview in enumerate(batch_overviews)])
+                                    final_prompt = f"""Below are multiple overview summaries, each covering different aspects of CGIAR research results. Each part maintains its original references to source documents.
+
+Please create a final comprehensive synthesis that:
+1. Integrates the key themes and findings from all parts into a cohesive narrative
+2. Preserves all hyperlinked references exactly as they appear
+3. Maintains the HTML anchor tags (<a href="...">) intact
+4. Uses clear section headings and structured formatting
+5. Highlights cross-cutting themes and relationships between different aspects
+6. Provides a clear introduction and conclusion
+
+Here are the overviews to synthesize:
+
+# Part 1
+
+{combined_overviews}"""
+                                    
+                                    final_prompt_tokens = len(tokenizer(final_prompt)["input_ids"])
+                                    if final_prompt_tokens > MAX_SAFE_TOKENS:
+                                        # If too long, just combine with headers
+                                        high_level_summary = "# Comprehensive Overview\n\n" + "\n\n# Part ".join([f"{i+1}\n\n{overview}" for i, overview in enumerate(batch_overviews)])
+                                    else:
+                                        high_level_chain = LLMChain(llm=llm, prompt=local_chat_prompt)
+                                        high_level_summary = high_level_chain.run(user_prompt=final_prompt).strip()
+                                else:
+                                    # If only one batch, use its overview directly
+                                    high_level_summary = batch_overviews[0]
+                                
+                                st.session_state['high_level_summary'] = high_level_summary
+                                st.session_state['enhanced_summary'] = high_level_summary
+                                
+                                # Display summaries
+                                st.write("### High-Level Summary:")
+                                with st.expander("High-Level Summary", expanded=True):
+                                    st.markdown(high_level_summary, unsafe_allow_html=True)
+
+                                st.write("### Cluster Summaries:")
+                                for idx, row in summary_df.iterrows():
+                                    cluster_name = row.get('Cluster_Name', 'Unnamed Cluster')
+                                    with st.expander(f"Topic {row['Topic']} - {cluster_name}", expanded=False):
+                                        st.markdown(row.get('Enhanced_Summary', row['Summary']), unsafe_allow_html=True)
+    st.markdown("##### About this tool")
+    with st.expander("Click to expand/collapse", expanded=True):
+        st.markdown("""
+        This tool draws on CGIAR quality assured results data from 2022-2024 to provide verifiable responses to user questions around the themes and areas CGIAR has/is working on.
+                    
+        **Tips:**
+        - **Craft a phrase** that describes your topic of interest (e.g., `"climate-smart agriculture"`, `"gender equality livestock"`).
+        - Avoid writing full questions — **this is not a chatbot**.
+        - Combine **related terms** for better results (e.g., `"irrigation water access smallholders"`).
+        - Focus on **concepts or themes** — not single words like `"climate"` or `"yield"` alone.
+        - Example good queries:
+            - `"climate adaptation smallholder farming"`
+            - `"digital agriculture innovations"`
+            - `"nutrition-sensitive value chains"`
+
+        **Example use case**:
+        You're interested in CGIAR's contributions to **poverty reduction through improved maize varieties in Africa**.  
+        A good search phrase would be:  
+        👉 `"poverty reduction maize Africa"`  
+        This will retrieve results related to improved crop varieties, livelihood outcomes, and region-specific interventions, even if the documents use different wording like *"enhanced maize genetics"*, *"smallholder income"*, or *"eastern Africa trials"*.
+        """)
\ No newline at end of file