refactor: Modularize application into single-responsibility components

Decompose the monolithic app.py into a clean, modular architecture to improve maintainability, readability, and scalability. Each new
module now has a clearly defined purpose, separating concerns like configuration, data, connections, and UI.

Key changes:
- `app.py`: Transformed into a lean entry point that initializes services and launches the UI.
- `config.py`: Created to manage all environment variables and configuration constants.
- `solr_metadata.py`: Isolates the static Solr field definitions, removing a large data structure from the main logic.
- `connections.py`: Centralizes the setup and initialization of external services (SSH Tunnel, Solr, Gemini LLM).
- `llm_prompts.py`: Contains all large f-string prompts for the LLM, cleaning up the data processing logic.
- `data_processing.py`: Holds the core data analysis functions for querying Solr and orchestrating the LLM.
- `ui.py`: Defines the Gradio user interface and its associated event-handling logic.

app.py

@@ -1,813 +1,42 @@
-import gradio as gr
-import json
-import re
-import datetime
-import pandas as pd
-import pysolr
-import google.generativeai as genai
-from sshtunnel import SSHTunnelForwarder
-import matplotlib.pyplot as plt
-import seaborn as sns
-import io
-import os
+"""
+Main entry point for the PharmaCircle AI Data Analyst application.
+This script initializes the necessary services and launches the Gradio user interface.
+It has been refactored to be a lean entry point, delegating all complex logic
+to specialized modules.
+"""
 import logging
-import concurrent.futures
-from IPython.display import display, Markdown
-import copy
-
+import connections
+from ui import create_ui
 
 # --- Suppress Matplotlib Debug Logs ---
 logging.getLogger('matplotlib').setLevel(logging.WARNING)
 
-# --- SSH Tunnel Configuration ---
-# It's recommended to load secrets securely, e.g., from environment variables
-SSH_HOST = os.environ.get('SSH_HOST')
-SSH_PORT = 5322
-SSH_USER = os.environ.get('SSH_USER')
-SSH_PASS = os.environ.get('SSH_PASS')
-
-# --- Solr Configuration ---
-REMOTE_SOLR_HOST = '69.167.186.48'
-REMOTE_SOLR_PORT = 8983
-LOCAL_BIND_PORT = 8983
-SOLR_CORE_NAME = 'news'
-SOLR_USER = os.environ.get('SOLR_USER')
-SOLR_PASS = os.environ.get('SOLR_PASS')
-
-# --- Google Gemini Configuration ---
-try:
-    genai.configure(api_key=os.environ.get('GEMINI_API_KEY'))
-except Exception as e:
-    print(f"❌ Gemini API Key Error: {e}. Please ensure 'GEMINI_API_KEY' is set in your environment.")
-
-# --- Global Variables ---
-ssh_tunnel_server = None
-solr_client = None
-llm_model = None
-is_initialized = False
-
-try:
-    # 1. Start the SSH Tunnel
-    ssh_tunnel_server = SSHTunnelForwarder(
-        (SSH_HOST, SSH_PORT),
-        ssh_username=SSH_USER,
-        ssh_password=SSH_PASS,
-        remote_bind_address=(REMOTE_SOLR_HOST, REMOTE_SOLR_PORT),
-        local_bind_address=('127.0.0.1', LOCAL_BIND_PORT)
-    )
-    ssh_tunnel_server.start()
-    print(f"🚀 SSH tunnel established: Local Port {ssh_tunnel_server.local_bind_port} -> Remote Solr.")
-
-    # 2. Initialize the pysolr client
-    solr_url = f'http://127.0.0.1:{ssh_tunnel_server.local_bind_port}/solr/{SOLR_CORE_NAME}'
-    solr_client = pysolr.Solr(solr_url, auth=(SOLR_USER, SOLR_PASS), always_commit=True)
-    solr_client.ping()
-    print(f"✅ Solr connection successful on core '{SOLR_CORE_NAME}'.")
-
-    # 3. Initialize the LLM
-    llm_model = genai.GenerativeModel('gemini-2.5-flash', generation_config=genai.types.GenerationConfig(temperature=0))
-    print(f"✅ LLM Model '{llm_model.model_name}' initialized.")
-
-    print("✅ System Initialized Successfully.")
-    is_initialized = True
-
-except Exception as e:
-    print(f"\n❌ An error occurred during setup: {e}")
-    if ssh_tunnel_server and ssh_tunnel_server.is_active:
-        ssh_tunnel_server.stop()
-
-
-field_metadata = [
-    {
-        "field_name": "business_model",
-        "type": "string (categorical)",
-        "example_values": ["pharma/bio", "drug delivery", "pharma services"],
-        "definition": "The primary business category of the company involved in the news. Use for filtering by high-level industry segments."
-    },
-    {
-        "field_name": "news_type",
-        "type": "string (categorical)",
-        "example_values": ["product news", "financial news", "regulatory news"],
-        "definition": "The category of the news article itself (e.g., financial, regulatory, acquisition). Use for filtering by the type of event being reported."
-    },
-    {
-        "field_name": "event_type",
-        "type": "string (categorical)",
-        "example_values": ["phase 2", "phase 1", "pre clinical", "marketed"],
-        "definition": "The clinical or developmental stage of a product or event discussed in the article. Essential for queries about clinical trial phases."
-    },
-    {
-        "field_name": "source",
-        "type": "string (categorical)",
-        "example_values": ["Press Release", "PR Newswire", "Business Wire"],
-        "definition": "The original source of the news article, such as a newswire or official report."
-    },
-    {
-        "field_name": "company_name",
-        "type": "string (exact match, for faceting)",
-        "example_values": ["pfizer inc.", "astrazeneca plc", "roche"],
-        "definition": "The canonical, standardized name of a company. **Crucially, you MUST use this field for `terms` faceting** to group results by a unique company. Do NOT use this for searching."
-    },
-    {
-        "field_name": "company_name_s",
-        "type": "string (multi-valued, for searching)",
-        "example_values": ["pfizer inc.", "roche", "f. hoffmann-la roche ag", "nih"],
-        "definition": "A field containing all known names and synonyms for a company. **You MUST use this field for all `query` parameter searches involving a company name** to ensure comprehensive results. Do NOT use for `terms` faceting."
-    },
-    {
-        "field_name": "territory_hq_s",
-        "type": "string (multi-valued, hierarchical)",
-        "example_values": ["united states of america", "europe", "europe western"],
-        "definition": "The geographic location (country and continent) of a company's headquarters. It is hierarchical. Use for filtering by location."
-    },
-    {
-        "field_name": "therapeutic_category",
-        "type": "string (specific)",
-        "example_values": ["cancer, other", "cancer, nsclc metastatic", "alzheimer's"],
-        "definition": "The specific disease or therapeutic area being targeted. Use for very specific disease queries."
-    },
-    {
-        "field_name": "therapeutic_category_s",
-        "type": "string (multi-valued, for searching)",
-        "example_values": ["cancer", "oncology", "infections", "cns"],
-        "definition": "Broader, multi-valued therapeutic categories and their synonyms. **Use this field for broad category searches** in the `query` parameter."
-    },
-    {
-        "field_name": "compound_name",
-        "type": "string (exact match, for faceting)",
-        "example_values": ["opdivo injection solution", "keytruda injection solution"],
-        "definition": "The specific, full trade name of a drug. **Use this field for `terms` faceting** on compounds."
-    },
-    {
-        "field_name": "compound_name_s",
-        "type": "string (multi-valued, for searching)",
-        "example_values": ["nivolumab injection solution", "opdivo injection solution", "ono-4538 injection solution"],
-        "definition": "A field with all known trade names and synonyms for a drug. **Use this field for all `query` parameter searches** involving a compound name."
-    },
-    {
-        "field_name": "molecule_name",
-        "type": "string (exact match, for faceting)",
-        "example_values": ["cannabidiol", "paclitaxel", "pembrolizumab"],
-        "definition": "The generic, non-proprietary name of the active molecule. **Use this field for `terms` faceting** on molecules."
-    },
-    {
-        "field_name": "molecule_name_s",
-        "type": "string (multi-valued, for searching)",
-        "example_values": ["cbd", "s1-220", "a1002n5s"],
-        "definition": "A field with all known generic names and synonyms for a molecule. **Use this field for all `query` parameter searches** involving a molecule name."
-    },
-    {
-        "field_name": "highest_phase",
-        "type": "string (categorical)",
-        "example_values": ["marketed", "phase 2", "phase 1"],
-        "definition": "The highest stage of development a drug has ever reached."
-    },
-    {
-        "field_name": "drug_delivery_branch_s",
-        "type": "string (multi-valued, for searching)",
-        "example_values": ["injection", "parenteral", "oral", "injection, other", "oral, other"],
-        "definition": "The method of drug administration. **Use this for `query` parameter searches about route of administration** as it contains broader, search-friendly terms."
-    },
-    {
-        "field_name": "drug_delivery_branch",
-        "type": "string (categorical, specific, for faceting)",
-        "example_values": ["injection, other", "prefilled syringes", "np liposome", "oral enteric/delayed release"],
-        "definition": "The most specific category of drug delivery technology. **Use this field for `terms` faceting** on specific delivery technologies."
-    },
-    {
-        "field_name": "route_branch",
-        "type": "string (categorical)",
-        "example_values": ["injection", "oral", "topical", "inhalation"],
-        "definition": "The primary route of drug administration. Good for faceting on exact routes."
-    },
-    {
-        "field_name": "molecule_api_group",
-        "type": "string (categorical)",
-        "example_values": ["small molecules", "biologics", "nucleic acids"],
-        "definition": "High-level classification of the drug's molecular type."
-    },
-    {
-        "field_name": "content",
-        "type": "text (full-text search)",
-        "example_values": ["The largest study to date...", "balstilimab..."],
-        "definition": "The full text content of the news article. Use for keyword searches on topics not covered by other specific fields."
-    },
-    {
-        "field_name": "date",
-        "type": "date",
-        "example_values": ["2020-10-22T00:00:00Z"],
-        "definition": "The full publication date and time in ISO 8601 format. Use for precise date range queries."
-    },
-    {
-        "field_name": "date_year",
-        "type": "number (year)",
-        "example_values": [2020, 2021, 2022],
-        "definition": "The 4-digit year of publication. **Use this for queries involving whole years** (e.g., 'in 2023', 'last year', 'since 2020')."
-    },
-    {
-        "field_name": "total_deal_value_in_million",
-        "type": "number (metric)",
-        "example_values": [50, 120.5, 176.157, 1000],
-        "definition": "The total value of a financial deal, in millions of USD. This is the primary numeric field for financial aggregations (sum, avg, etc.). To use this, you must also filter for news that has a deal value, e.g., 'total_deal_value_in_million:[0 TO *]'."
-    }
-]
-
-# Helper function to format the metadata for the prompt
-def format_metadata_for_prompt(metadata):
-    formatted_string = ""
-    for field in metadata:
-        formatted_string += f"- **{field['field_name']}**\n"
-        formatted_string += f"  - **Type**: {field['type']}\n"
-        formatted_string += f"  - **Definition**: {field['definition']}\n"
-        formatted_string += f"  - **Examples**: {', '.join(map(str, field['example_values']))}\n\n"
-    return formatted_string
-formatted_field_info = format_metadata_for_prompt(field_metadata)
-
-
-def parse_suggestions_from_report(report_text):
-    """Extracts numbered suggestions from the report's markdown text."""
-    suggestions_match = re.search(r"### (?:Deeper Dive: Suggested Follow-up Analyses|Suggestions for Further Exploration)\s*\n(.*?)$", report_text, re.DOTALL | re.IGNORECASE)
-    if not suggestions_match: return []
-    suggestions_text = suggestions_match.group(1)
-    suggestions = re.findall(r"^\s*\d+\.\s*(.*)", suggestions_text, re.MULTILINE)
-    return [s.strip() for s in suggestions]
-
-
-def llm_generate_analysis_plan_with_history(natural_language_query, field_metadata, chat_history):
+def main():
     """
-    Generates a complete analysis plan from a user query, considering chat history.
-    This plan includes dimensions, measures, and requests for both quantitative (
-facet)
-    and qualitative (grouping) data.
+    Initializes connections and launches the Gradio application.
     """
-    formatted_history = ""
-    for user_msg, bot_msg in chat_history:
-        if user_msg:
-            formatted_history += f"- User: \"{user_msg}\"\n"
-
-    prompt = f"""
-You are an expert data analyst and Solr query engineer. Your task is to convert a natural language question into a structured JSON "Analysis Plan". This plan will be used to run two separate, efficient queries: one for aggregate data (facets) and one for finding illustrative examples (grouping).
-
----
-### CONTEXT & RULES
-
-1.  **Today's Date for Calculations**: {datetime.datetime.now().date().strftime("%Y-%m-%d")}
-2.  **Field Usage**: You MUST use the fields described in the 'Field Definitions'. Pay close attention to the definitions to select the correct field, especially the `_s` fields for searching. Do not use fields ending with `_s` in `group.field` or facet `field` unless necessary for the analysis.
-3.  **Dimension vs. Measure**:
-    *   `analysis_dimension`: The primary categorical field the user wants to group by (e.g., `company_name`, `route_branch`). This is the `group by` field.
-    *   `analysis_measure`: The metric to aggregate (e.g., `sum(total_deal_value_in_million)`) or the method of counting (`count`).
-    *   `sort_field_for_examples`: The raw field used to find the "best" example. If `analysis_measure` is `sum(field)`, this should be `field`. If `analysis_measure` is `count`, this should be a relevant field like `date`.
-4.  **Crucial Sorting Rules**:
-    *   For `group.sort`: If `analysis_measure` involves a function on a field (e.g., `sum(total_deal_value_in_million)`), you MUST use the full function: `group.sort: 'sum(total_deal_value_in_million) desc'`.
-    *   If `analysis_measure` is 'count', you MUST OMIT the `group.sort` parameter entirely.
-    *   For sorting, NEVER use 'date_year'; use 'date' instead.
-5.  **Output Format**: Your final output must be a single, raw JSON object. Do not add comments or markdown formatting.
-
----
-### FIELD DEFINITIONS (Your Source of Truth)
-
-{formatted_field_info}
----
-### CHAT HISTORY
-{formatted_history}
----
-### EXAMPLES
-
-**User Query 1:** "What are the top 5 companies by total deal value in 2023?"
-**Correct JSON Output 1:**
-```json
-{{
-  "analysis_dimension": "company_name",
-  "analysis_measure": "sum(total_deal_value_in_million)",
-  "sort_field_for_examples": "total_deal_value_in_million",
-  "query_filter": "date_year:2023 AND total_deal_value_in_million:[0 TO *]",
-  "quantitative_request": {{
-    "json.facet": {{
-      "companies_by_deal_value": {{
-        "type": "terms",
-        "field": "company_name",
-        "limit": 5,
-        "sort": "total_value desc",
-        "facet": {{
-          "total_value": "sum(total_deal_value_in_million)"
-        }}
-      }}
-    }}
-  }},
-  "qualitative_request": {{
-    "group": true,
-    "group.field": "company_name",
-    "group.limit": 1,
-    "group.sort": "sum(total_deal_value_in_million) desc",
-    "sort": "total_deal_value_in_million desc"
-  }}
-}}
-```
-
-**User Query 2:** "What are the most common news types for infections this year?"
-**Correct JSON Output 2:**
-```json
-{{
-  "analysis_dimension": "news_type",
-  "analysis_measure": "count",
-  "sort_field_for_examples": "date",
-  "query_filter": "therapeutic_category_s:infections AND date_year:{datetime.datetime.now().year}",
-  "quantitative_request": {{
-    "json.facet": {{
-      "news_by_type": {{
-        "type": "terms",
-        "field": "news_type",
-        "limit": 10,
-        "sort": "count desc"
-      }}
-    }}
-  }},
-  "qualitative_request": {{
-    "group": true,
-    "group.field": "news_type",
-    "group.limit": 1,
-    "sort": "date desc"
-  }}
-}}
-```
----
-### YOUR TASK
-
-Convert the following user query into a single, raw JSON "Analysis Plan" object, strictly following all rules and considering the chat history.
-
-**Current User Query:** `{natural_language_query}`
-"""
-    try:
-        response = llm_model.generate_content(prompt)
-        cleaned_text = re.sub(r'```json\s*|\s*```', '', response.text, flags=re.MULTILINE | re.DOTALL).strip()
-        plan = json.loads(cleaned_text)
-        return plan
-    except Exception as e:
-        raw_response_text = response.text if 'response' in locals() else 'N/A'
-        print(f"Error in llm_generate_analysis_plan_with_history: {e}\nRaw Response:\n{raw_response_text}")
-        return None
-
-def execute_quantitative_query(plan, solr):
-    """Executes the facet query to get aggregate data."""
-    if not plan or 'quantitative_request' not in plan or 'json.facet' not in plan.get('quantitative_request', {}):
-        return None
-    try:
-        params = {
-            "q": plan.get('query_filter', '*:*'),
-            "rows": 0,
-            "json.facet": json.dumps(plan['quantitative_request']['json.facet'])
-        }
-        results = solr.search(**params)
-        return results.raw_response.get("facets", {})
-    except Exception as e:
-        print(f"Error in quantitative query: {e}")
-        return None
-
-def execute_qualitative_query(plan, solr):
-    """Executes the grouping query to get the best example docs."""
-    if not plan or 'qualitative_request' not in plan:
-        return None
-    try:
-        qual_request = copy.deepcopy(plan['qualitative_request'])
-        params = {
-            "q": plan.get('query_filter', '*:*'),
-            "rows": 3, # Get a few examples per group
-            "fl": "*,score",
-            **qual_request
-        }
-        results = solr.search(**params)
-        return results.grouped
-    except Exception as e:
-        print(f"Error in qualitative query: {e}")
-        return None
-
-def llm_synthesize_enriched_report_stream(query, quantitative_data, qualitative_data, plan):
-    """
-    Generates an enriched report by synthesizing quantitative aggregates
-    and qualitative examples, and streams the result.
-    """
-    qualitative_prompt_str = ""
-    dimension = plan.get('analysis_dimension', 'N/A')
-    if qualitative_data and dimension in qualitative_data:
-        for group in qualitative_data.get(dimension, {}).get('groups', []):
-            group_value = group.get('groupValue', 'N/A')
-            if group.get('doclist', {}).get('docs'):
-                doc = group.get('doclist', {}).get('docs', [{}])[0]
-                title = doc.get('abstract', ['No Title'])
-                content_list = doc.get('content', [])
-                content_snip = (' '.join(content_list[0].split()[:40]) + '...') if content_list else 'No content available.'
-                metric_val_raw = doc.get(plan.get('sort_field_for_examples'), 'N/A')
-                metric_val = metric_val_raw[0] if isinstance(metric_val_raw, list) else metric_val_raw
-
-                qualitative_prompt_str += f"- **For category `{group_value}`:**\n"
-                qualitative_prompt_str += f"  - **Top Example Title:** {title}\n"
-                qualitative_prompt_str += f"  - **Metric Value:** {metric_val}\n"
-                qualitative_prompt_str += f"  - **Content Snippet:** {content_snip}\n\n"
-
-    prompt = f"""
-You are a top-tier business intelligence analyst. Your task is to write an insightful, data-driven report for an executive. You must synthesize quantitative data (the 'what') with qualitative examples (the 'why') to tell a complete story.
-
----
-### AVAILABLE INFORMATION
-
-**1. The User's Core Question:**
-\"{query}\"
-
-**2. Quantitative Data (The 'What'):**
-This data shows the high-level aggregates.
-```json
-{json.dumps(quantitative_data, indent=2)}
-```
-
-**3. Qualitative Data (The 'Why'):**
-These are the single most significant documents driving the numbers for each category.
-{qualitative_prompt_str}
-
----
-### REPORTING INSTRUCTIONS
-
-Your report must be in clean, professional Markdown and follow this structure precisely.
-
-**Report Structure:**
-
-`## Executive Summary`
-- A 1-2 sentence, top-line answer to the user's question based on the quantitative data.
-
-`### Key Findings`
-- Use bullet points to highlight the main figures from the quantitative data. Interpret the numbers.
-
-`### Key Drivers & Illustrative Examples`
-- **This is the most important section.** Explain the "so what?" behind the numbers.
-- Use the qualitative examples to explain *why* a category is high or low. Reference the top example document for each main category.
-
-`### Deeper Dive: Suggested Follow-up Analyses`
-- Propose 2-3 logical next questions based on your analysis to uncover deeper trends.
-
----
-**Generate the full report now, paying close attention to all formatting and spacing rules.**
-"""
-    try:
-        response_stream = llm_model.generate_content(prompt, stream=True)
-        for chunk in response_stream:
-            yield chunk.text
-    except Exception as e:
-        print(f"Error in llm_synthesize_enriched_report_stream: {e}")
-        yield "Sorry, I was unable to generate a report for this data."
-
-
-def llm_generate_visualization_code(query_context, facet_data):
-    """Generates Python code for visualization based on query and data."""
-    prompt = f"""
-You are a Python Data Visualization expert specializing in Matplotlib and Seaborn.
-Your task is to generate robust, error-free Python code to create a single, insightful visualization based on the user's query and the provided Solr facet data.
-
-**User's Analytical Goal:**
-\"{query_context}\"
-
-**Aggregated Data (from Solr Facets):**
-```json
-{json.dumps(facet_data, indent=2)}
-```
-
----
-### **CRITICAL INSTRUCTIONS: CODE GENERATION RULES**
-You MUST follow these rules to avoid errors.
-
-**1. Identify the Data Structure FIRST:**
-Before writing any code, analyze the `facet_data` JSON to determine its structure. There are three common patterns. Choose the correct template below.
-
-   *   **Pattern A: Simple `terms` Facet.** The JSON has ONE main key (besides "count") which contains a list of "buckets". Each bucket has a "val" and a "count". Use this for standard bar charts.
-   *   **Pattern B: Multiple `query` Facets.** The JSON has MULTIPLE keys (besides "count"), and each key is an object containing metrics like "count" or "sum(...)". Use this for comparing a few distinct items (e.g., "oral vs injection").
-   *   **Pattern C: Nested `terms` Facet.** The JSON has one main key with a list of "buckets", but inside EACH bucket, there are nested metric objects. This is used for grouped comparisons (e.g., "compare 2024 vs 2025 across categories"). This almost always requires `pandas`.
-
-**2. Use the Correct Parsing Template:**
-
----
-**TEMPLATE FOR PATTERN A (Simple Bar Chart from `terms` facet):**
-```python
-import matplotlib.pyplot as plt
-import seaborn as sns
-import pandas as pd
-
-plt.style.use('seaborn-v0_8-whitegrid')
-fig, ax = plt.subplots(figsize=(12, 8))
-
-# Dynamically find the main facet key (the one with 'buckets')
-facet_key = None
-for key, value in facet_data.items():
-    if isinstance(value, dict) and 'buckets' in value:
-        facet_key = key
-        break
-
-if facet_key:
-    buckets = facet_data[facet_key].get('buckets', [])
-    # Check if buckets contain data
-    if buckets:
-        df = pd.DataFrame(buckets)
-        # Check for a nested metric or use 'count'
-        if 'total_deal_value' in df.columns and pd.api.types.is_dict_like(df['total_deal_value'].iloc):
-             # Example for nested sum metric
-             df['value'] = df['total_deal_value'].apply(lambda x: x.get('sum', 0))
-             y_axis_label = 'Sum of Total Deal Value'
-        else:
-             df.rename(columns={{'count': 'value'}}, inplace=True)
-             y_axis_label = 'Count'
-
-        sns.barplot(data=df, x='val', y='value', ax=ax, palette='viridis')
-        ax.set_xlabel('Category')
-        ax.set_ylabel(y_axis_label)
-    else:
-        ax.text(0.5, 0.5, 'No data in buckets to plot.', ha='center')
-
-
-ax.set_title('Your Insightful Title Here')
-# Correct way to rotate labels to prevent errors
-plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
-plt.tight_layout()
-```
----
-**TEMPLATE FOR PATTERN B (Comparison Bar Chart from `query` facets):**
-```python
-import matplotlib.pyplot as plt
-import seaborn as sns
-import pandas as pd
-
-plt.style.use('seaborn-v0_8-whitegrid')
-fig, ax = plt.subplots(figsize=(10, 6))
-
-labels = []
-values = []
-# Iterate through top-level keys, skipping the 'count'
-for key, data_dict in facet_data.items():
-    if key == 'count' or not isinstance(data_dict, dict):
-        continue
-    # Extract the label (e.g., 'oral_deals' -> 'Oral')
-    label = key.replace('_deals', '').replace('_', ' ').title()
-    # Find the metric value, which is NOT 'count'
-    metric_value = 0
-    for sub_key, sub_value in data_dict.items():
-        if sub_key != 'count':
-            metric_value = sub_value
-            break # Found the metric
-    labels.append(label)
-    values.append(metric_value)
-
-if labels:
-    sns.barplot(x=labels, y=values, ax=ax, palette='mako')
-    ax.set_ylabel('Total Deal Value') # Or other metric name
-    ax.set_xlabel('Category')
-else:
-    ax.text(0.5, 0.5, 'No query facet data to plot.', ha='center')
-
-
-ax.set_title('Your Insightful Title Here')
-plt.tight_layout()
-```
----
-**TEMPLATE FOR PATTERN C (Grouped Bar Chart from nested `terms` facet):**
-```python
-import matplotlib.pyplot as plt
-import seaborn as sns
-import pandas as pd
-
-plt.style.use('seaborn-v0_8-whitegrid')
-fig, ax = plt.subplots(figsize=(14, 8))
-
-# Find the key that has the buckets
-facet_key = None
-for key, value in facet_data.items():
-    if isinstance(value, dict) and 'buckets' in value:
-        facet_key = key
-        break
-
-if facet_key and facet_data[facet_key].get('buckets'):
-    # This list comprehension is robust for parsing nested metrics
-    plot_data = []
-    for bucket in facet_data[facet_key]['buckets']:
-        category = bucket['val']
-        # Find all nested metrics (e.g., total_deal_value_2025)
-        for sub_key, sub_value in bucket.items():
-            if isinstance(sub_value, dict) and 'sum' in sub_value:
-                # Extracts year from 'total_deal_value_2025' -> '2025'
-                year = sub_key.split('_')[-1]
-                value = sub_value['sum']
-                plot_data.append({{'Category': category, 'Year': year, 'Value': value}})
-
-    if plot_data:
-        df = pd.DataFrame(plot_data)
-        sns.barplot(data=df, x='Category', y='Value', hue='Year', ax=ax)
-        ax.set_ylabel('Total Deal Value')
-        ax.set_xlabel('Business Model')
-        # Correct way to rotate labels to prevent errors
-        plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
-    else:
-        ax.text(0.5, 0.5, 'No nested data found to plot.', ha='center')
-else:
-    ax.text(0.5, 0.5, 'No data in buckets to plot.', ha='center')
-
-ax.set_title('Your Insightful Title Here')
-plt.tight_layout()
-```
----
-**3. Final Code Generation:**
-- **DO NOT** include `plt.show()`.
-- **DO** set a dynamic and descriptive `ax.set_title()`, `ax.set_xlabel()`, and `ax.set_ylabel()`.
-- **DO NOT** wrap the code in ```python ... ```. Output only the raw Python code.
-- Adapt the chosen template to the specific keys and metrics in the provided `facet_data`.
-
-**Your Task:**
-Now, generate the Python code.
-"""
-    try:
-        # Increase the timeout for potentially complex generation
-        generation_config = genai.types.GenerationConfig(temperature=0, max_output_tokens=2048)
-        response = llm_model.generate_content(prompt, generation_config=generation_config)
-        # Clean the response to remove markdown formatting
-        code = re.sub(r'^```python\s*|```$', '', response.text, flags=re.MULTILINE)
-        return code
-    except Exception as e:
-        print(f"Error in llm_generate_visualization_code: {e}\nRaw response: {response.text}")
-        return None
+    # Initialize all external connections (SSH, Solr, LLM)
+    ssh_tunnel, solr_client, llm_model = connections.initialize_connections()
+
+    if not all([ssh_tunnel, solr_client, llm_model]):
+        print("\nSkipping Gradio launch due to initialization errors.")
+        # Ensure the tunnel is closed if it was partially opened
+        if ssh_tunnel and ssh_tunnel.is_active:
+            ssh_tunnel.stop()
+        return
 
-def execute_viz_code_and_get_path(viz_code, facet_data):
-    """Executes visualization code and returns the path to the saved plot image."""
-    if not viz_code: return None
+    # Create and launch the Gradio UI
+    demo = create_ui(llm_model, solr_client)
     try:
-        if not os.path.exists('/tmp/plots'): os.makedirs('/tmp/plots')
-        plot_path = f"/tmp/plots/plot_{datetime.datetime.now().timestamp()}.png"
-        # The exec environment needs access to the required libraries and the data
-        exec_globals = {'facet_data': facet_data, 'plt': plt, 'sns': sns, 'pd': pd}
-        exec(viz_code, exec_globals)
-        fig = exec_globals.get('fig')
-        if fig:
-            fig.savefig(plot_path, bbox_inches='tight')
-            plt.close(fig) # Important to free up memory
-            return plot_path
-        return None
+        demo.queue().launch(debug=True, share=True)
     except Exception as e:
-        print(f"ERROR executing visualization code: {e}\n---Code---\n{viz_code}")
-        return None
-
-
-def process_analysis_flow(user_input, history, state):
-    """
-    A generator that manages the conversation and yields tuples of UI updates for Gradio.
-    This version uses the dual-query (quantitative/qualitative) approach.
-    """
-    if state is None:
-        state = {'query_count': 0, 'last_suggestions': []}
-    if history is None:
-        history = []
-
-    # Reset UI for new analysis
-    yield (history, state, gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False))
-
-    query_context = user_input.strip()
-    if not query_context:
-        history.append((user_input, "Please enter a question to analyze."))
-        yield (history, state, None, None, None, None, None)
-        return
-
-    # 1. Acknowledge and generate plan
-    history.append((user_input, f"Analyzing: '{query_context}'\n\n*Generating analysis plan...*"))
-    yield (history, state, None, None, None, None, None)
-
-    analysis_plan = llm_generate_analysis_plan_with_history(query_context, field_metadata, history)
-    if not analysis_plan:
-        history.append((None, "I'm sorry, I couldn't generate a valid analysis plan for that request. Please try rephrasing."))
-        yield (history, state, None, None, None, None, None)
-        return
-
-    history.append((None, "✅ Analysis plan generated!"))
-    plan_summary = f"""
-*   **Analysis Dimension:** `{analysis_plan.get('analysis_dimension')}`
-*   **Analysis Measure:** `{analysis_plan.get('analysis_measure')}`
-*   **Query Filter:** `{analysis_plan.get('query_filter')}`
-"""
-    # Show the plan summary in the main chat
-    history.append((None, plan_summary))
-    # Put the full plan in the accordion
-    formatted_plan = f"**Full Analysis Plan:**\n```json\n{json.dumps(analysis_plan, indent=2)}\n```"
-    yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
-
-
-    # 2. Execute Queries in Parallel
-    history.append((None, "*Executing queries for aggregates and examples...*"))
-    yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
-
-    aggregate_data = None
-    example_data = None
-    with concurrent.futures.ThreadPoolExecutor() as executor:
-        future_agg = executor.submit(execute_quantitative_query, analysis_plan, solr_client)
-        future_ex = executor.submit(execute_qualitative_query, analysis_plan, solr_client)
-        aggregate_data = future_agg.result()
-        example_data = future_ex.result()
-
-    if not aggregate_data or aggregate_data.get('count', 0) == 0:
-        history.append((None, "No data was found for your query. Please try a different question."))
-        yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
-        return
-
-    # Display retrieved data in accordions
-    formatted_agg_data = f"**Quantitative (Aggregate) Data:**\n```json\n{json.dumps(aggregate_data, indent=2)}\n```"
-    formatted_qual_data = f"**Qualitative (Example) Data:**\n```json\n{json.dumps(example_data, indent=2)}\n```"
-    qual_data_display_update = gr.update(value=formatted_qual_data, visible=True)
-    yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
-
-
-    # 3. Generate Visualization (in parallel with report)
-    history.append((None, "✅ Data retrieved. Generating visualization and final report..."))
-    yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
-
-    with concurrent.futures.ThreadPoolExecutor() as executor:
-        viz_future = executor.submit(llm_generate_visualization_code, query_context, aggregate_data)
-
-        # 4. Generate and Stream Enriched Report
-        report_text = ""
-        stream_history = history[:]
-        for chunk in llm_synthesize_enriched_report_stream(query_context, aggregate_data, example_data, analysis_plan):
-            report_text += chunk
-            yield (stream_history, state, None, gr.update(value=report_text, visible=True), gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
-
-        history.append((None, report_text))
-
-        # Get visualization from future
-        viz_code = viz_future.result()
-        plot_path = execute_viz_code_and_get_path(viz_code, aggregate_data)
-        output_plot = gr.update(value=plot_path, visible=True) if plot_path else gr.update(visible=False)
-        if not plot_path:
-            history.append((None, "*I was unable to generate a plot for this data.*\n"))
-
-        yield (history, state, output_plot, report_text, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
-
-    # 5. Finalize
-    state['query_count'] += 1
-    state['last_suggestions'] = parse_suggestions_from_report(report_text)
-    next_prompt = "Analysis complete. What would you like to explore next?"
-    history.append((None, next_prompt))
-    yield (history, state, output_plot, report_text, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
-
-
-# --- Gradio UI ---
-with gr.Blocks(theme=gr.themes.Soft(), css="footer {display: none !important}") as demo:
-    state = gr.State()
-
-    with gr.Row():
-        with gr.Column(scale=4):
-            gr.Markdown("# 💊 PharmaCircle AI Data Analyst")
-        with gr.Column(scale=1):
-            clear_button = gr.Button("🔄 Start New Analysis", variant="primary")
-
-    gr.Markdown("Ask a question to begin your analysis. I will generate an analysis plan, retrieve quantitative and qualitative data, create a visualization, and write an enriched report.")
-
-    with gr.Row():
-        with gr.Column(scale=1):
-            chatbot = gr.Chatbot(label="Analysis Chat Log", height=700, show_copy_button=True)
-            msg_textbox = gr.Textbox(placeholder="Ask a question, e.g., 'Show me the top 5 companies by total deal value in 2023'", label="Your Question", interactive=True)
-
-        with gr.Column(scale=2):
-            with gr.Accordion("Generated Analysis Plan", open=False):
-                plan_display = gr.Markdown("Plan will appear here...", visible=True)
-            with gr.Accordion("Retrieved Quantitative Data", open=False):
-                quantitative_data_display = gr.Markdown("Aggregate data will appear here...", visible=False)
-            with gr.Accordion("Retrieved Qualitative Data (Examples)", open=False):
-                qualitative_data_display = gr.Markdown("Example data will appear here...", visible=False)
-            plot_display = gr.Image(label="Visualization", type="filepath", visible=False)
-            report_display = gr.Markdown("Report will be streamed here...", visible=False)
-
-    # --- Event Wiring ---
-    def reset_all():
-        """Resets the entire UI for a new analysis session."""
-        return (
-            [],   # chatbot
-            None, # state
-            "",   # msg_textbox
-            gr.update(value=None, visible=False), # plot_display
-            gr.update(value=None, visible=False), # report_display
-            gr.update(value=None, visible=False), # plan_display
-            gr.update(value=None, visible=False), # quantitative_data_display
-            gr.update(value=None, visible=False)  # qualitative_data_display
-        )
-
-    msg_textbox.submit(
-        fn=process_analysis_flow,
-        inputs=[msg_textbox, chatbot, state],
-        outputs=[chatbot, state, plot_display, report_display, plan_display, quantitative_data_display, qualitative_data_display],
-    ).then(
-        lambda: gr.update(value=""),
-        None,
-        [msg_textbox],
-        queue=False,
-    )
-
-    clear_button.click(
-        fn=reset_all,
-        inputs=None,
-        outputs=[chatbot, state, msg_textbox, plot_display, report_display, plan_display, quantitative_data_display, qualitative_data_display],
-        queue=False
-    )
-
-if is_initialized:
-    demo.queue().launch(debug=True, share=True)
-else:
-    print("\nSkipping Gradio launch due to initialization errors.")
+        print(f"An error occurred while launching the Gradio app: {e}")
+    finally:
+        # Ensure the SSH tunnel is closed when the app is shut down
+        print("\nClosing SSH tunnel...")
+        if ssh_tunnel.is_active:
+            ssh_tunnel.stop()
+        print("SSH tunnel closed. Exiting.")
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,28 @@
+"""
+Configuration management for the PharmaCircle AI Data Analyst.
+
+Loads all necessary secrets and configuration parameters from environment variables.
+This approach keeps sensitive data out of the source code and allows for flexible deployment.
+"""
+
+import os
+
+# --- SSH Tunnel Configuration ---
+# Credentials for establishing the SSH tunnel to the Solr server environment.
+SSH_HOST = os.environ.get('SSH_HOST')
+SSH_PORT = 5322
+SSH_USER = os.environ.get('SSH_USER')
+SSH_PASS = os.environ.get('SSH_PASS')
+
+# --- Solr Configuration ---
+# Details for the remote Solr instance and the local port for the tunnel.
+REMOTE_SOLR_HOST = '69.167.186.48'
+REMOTE_SOLR_PORT = 8983
+LOCAL_BIND_PORT = 8983
+SOLR_CORE_NAME = 'news'
+SOLR_USER = os.environ.get('SOLR_USER')
+SOLR_PASS = os.environ.get('SOLR_PASS')
+
+# --- Google Gemini Configuration ---
+# API key for accessing the Google Gemini large language model.
+GEMINI_API_KEY = os.environ.get('GEMINI_API_KEY')

connections.py

@@ -0,0 +1,54 @@
+""" 
+Manages connections to external services: SSH, Solr, and Google Gemini.
+
+This module centralizes the initialization logic, making the main application
+cleaner and more focused on its primary task. It provides a single function
+to set up all necessary connections.
+"""
+
+
+import pysolr
+import google.generativeai as genai
+from sshtunnel import SSHTunnelForwarder
+import config
+
+def initialize_connections():
+    """
+    Establishes the SSH tunnel, and initializes Solr and Gemini clients.
+
+    Returns:
+        A tuple containing the initialized (ssh_tunnel_server, solr_client, llm_model).
+        Returns (None, None, None) if any part of the initialization fails.
+    """
+    ssh_tunnel_server = None
+    try:
+        # 1. Configure and start the SSH Tunnel
+        ssh_tunnel_server = SSHTunnelForwarder(
+            (config.SSH_HOST, config.SSH_PORT),
+            ssh_username=config.SSH_USER,
+            ssh_password=config.SSH_PASS,
+            remote_bind_address=(config.REMOTE_SOLR_HOST, config.REMOTE_SOLR_PORT),
+            local_bind_address=('127.0.0.1', config.LOCAL_BIND_PORT)
+        )
+        ssh_tunnel_server.start()
+        print(f"🚀 SSH tunnel established: Local Port {ssh_tunnel_server.local_bind_port} -> Remote Solr.")
+
+        # 2. Initialize the pysolr client
+        solr_url = f'http://127.0.0.1:{ssh_tunnel_server.local_bind_port}/solr/{config.SOLR_CORE_NAME}'
+        solr_client = pysolr.Solr(solr_url, auth=(config.SOLR_USER, config.SOLR_PASS), always_commit=True)
+        solr_client.ping()
+        print(f"✅ Solr connection successful on core '{config.SOLR_CORE_NAME}'.")
+
+        # 3. Initialize the LLM
+        genai.configure(api_key=config.GEMINI_API_KEY)
+        llm_model = genai.GenerativeModel('gemini-2.5-flash', generation_config=genai.types.GenerationConfig(temperature=0))
+        print(f"✅ LLM Model '{llm_model.model_name}' initialized.")
+
+        print("✅ System Initialized Successfully.")
+        return ssh_tunnel_server, solr_client, llm_model
+
+    except Exception as e:
+        print(f"\n❌ An error occurred during setup: {e}")
+        if ssh_tunnel_server and ssh_tunnel_server.is_active:
+            ssh_tunnel_server.stop()
+        return None, None, None

data_processing.py

@@ -0,0 +1,130 @@
+"""
+Core data processing and analysis logic for the PharmaCircle AI Data Analyst.
+
+This module orchestrates the main analysis workflow:
+1.  Takes a user's natural language query.
+2.  Uses the LLM to generate a structured analysis plan.
+3.  Executes parallel queries against Solr for quantitative and qualitative data.
+4.  Generates a data visualization using the LLM.
+5.  Synthesizes the findings into a comprehensive, user-facing report.
+"""
+
+
+import json
+import re
+import datetime
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os
+import concurrent.futures
+import copy
+import google.generativeai as genai
+
+from llm_prompts import (
+    get_analysis_plan_prompt,
+    get_synthesis_report_prompt,
+    get_visualization_code_prompt
+)
+
+def parse_suggestions_from_report(report_text):
+    """Extracts numbered suggestions from the report's markdown text."""
+    suggestions_match = re.search(r"### (?:Deeper Dive: Suggested Follow-up Analyses|Suggestions for Further Exploration)\s*\n(.*?)$", report_text, re.DOTALL | re.IGNORECASE)
+    if not suggestions_match: return []
+    suggestions_text = suggestions_match.group(1)
+    suggestions = re.findall(r"^\s*\d+\.\s*(.*)", suggestions_text, re.MULTILINE)
+    return [s.strip() for s in suggestions]
+
+def llm_generate_analysis_plan_with_history(llm_model, natural_language_query, chat_history):
+    """
+    Generates a complete analysis plan from a user query, considering chat history.
+    """
+    prompt = get_analysis_plan_prompt(natural_language_query, chat_history)
+    try:
+        response = llm_model.generate_content(prompt)
+        cleaned_text = re.sub(r'```json\s*|\s*```', '', response.text, flags=re.MULTILINE | re.DOTALL).strip()
+        plan = json.loads(cleaned_text)
+        return plan
+    except Exception as e:
+        raw_response_text = response.text if 'response' in locals() else 'N/A'
+        print(f"Error in llm_generate_analysis_plan_with_history: {e}\nRaw Response:\n{raw_response_text}")
+        return None
+
+def execute_quantitative_query(solr_client, plan):
+    """Executes the facet query to get aggregate data."""
+    if not plan or 'quantitative_request' not in plan or 'json.facet' not in plan.get('quantitative_request', {}):
+        return None
+    try:
+        params = {
+            "q": plan.get('query_filter', '*_*'),
+            "rows": 0,
+            "json.facet": json.dumps(plan['quantitative_request']['json.facet'])
+        }
+        results = solr_client.search(**params)
+        return results.raw_response.get("facets", {})
+    except Exception as e:
+        print(f"Error in quantitative query: {e}")
+        return None
+
+def execute_qualitative_query(solr_client, plan):
+    """Executes the grouping query to get the best example docs."""
+    if not plan or 'qualitative_request' not in plan:
+        return None
+    try:
+        qual_request = copy.deepcopy(plan['qualitative_request'])
+        params = {
+            "q": plan.get('query_filter', '*_*'),
+            "rows": 3,  # Get a few examples per group
+            "fl": "*,score",
+            **qual_request
+        }
+        results = solr_client.search(**params)
+        return results.grouped
+    except Exception as e:
+        print(f"Error in qualitative query: {e}")
+        return None
+
+def llm_synthesize_enriched_report_stream(llm_model, query, quantitative_data, qualitative_data, plan):
+    """
+    Generates an enriched report by synthesizing quantitative aggregates
+    and qualitative examples, and streams the result.
+    """
+    prompt = get_synthesis_report_prompt(query, quantitative_data, qualitative_data, plan)
+    try:
+        response_stream = llm_model.generate_content(prompt, stream=True)
+        for chunk in response_stream:
+            yield chunk.text
+    except Exception as e:
+        print(f"Error in llm_synthesize_enriched_report_stream: {e}")
+        yield "Sorry, I was unable to generate a report for this data."
+
+def llm_generate_visualization_code(llm_model, query_context, facet_data):
+    """Generates Python code for visualization based on query and data."""
+    prompt = get_visualization_code_prompt(query_context, facet_data)
+    try:
+        generation_config = genai.types.GenerationConfig(temperature=0, max_output_tokens=2048)
+        response = llm_model.generate_content(prompt, generation_config=generation_config)
+        code = re.sub(r'^```python\s*|```$', '', response.text, flags=re.MULTILINE)
+        return code
+    except Exception as e:
+        print(f"Error in llm_generate_visualization_code: {e}\nRaw response: {response.text}")
+        return None
+
+def execute_viz_code_and_get_path(viz_code, facet_data):
+    """Executes visualization code and returns the path to the saved plot image."""
+    if not viz_code: return None
+    try:
+        if not os.path.exists('/tmp/plots'): os.makedirs('/tmp/plots')
+        plot_path = f"/tmp/plots/plot_{datetime.datetime.now().timestamp()}.png"
+        exec_globals = {'facet_data': facet_data, 'plt': plt, 'sns': sns, 'pd': pd}
+        exec(viz_code, exec_globals)
+        fig = exec_globals.get('fig')
+        if fig:
+            fig.savefig(plot_path, bbox_inches='tight')
+            plt.close(fig)
+            return plot_path
+        return None
+    except Exception as e:
+        print(f"ERROR executing visualization code: {e}\n---Code---\n{viz_code}")
+        return None
+

llm_prompts.py

@@ -0,0 +1,346 @@
+"""
+Contains the prompt templates for interacting with the Gemini LLM.
+
+Separating prompts from the application logic makes them easier to manage,
+modify, and version. This module provides functions that return the formatted
+prompt strings required by the data processing module.
+"""
+
+
+import datetime
+import json
+from solr_metadata import format_metadata_for_prompt
+
+def get_analysis_plan_prompt(natural_language_query, chat_history):
+    """
+    Generates the prompt for creating a Solr analysis plan from a user query.
+    """
+    formatted_field_info = format_metadata_for_prompt()
+    formatted_history = ""
+    for user_msg, bot_msg in chat_history:
+        if user_msg:
+            formatted_history += f"- User: \"{user_msg}\"\n"
+
+    return f"""
+You are an expert data analyst and Solr query engineer. Your task is to convert a natural language question into a structured JSON "Analysis Plan". This plan will be used to run two separate, efficient queries: one for aggregate data (facets) and one for finding illustrative examples (grouping).
+
+---
+### CONTEXT & RULES
+
+1.  **Today's Date for Calculations**: {datetime.datetime.now().date().strftime("%Y-%m-%d")}
+2.  **Field Usage**: You MUST use the fields described in the 'Field Definitions'. Pay close attention to the definitions to select the correct field, especially the `_s` fields for searching. Do not use fields ending with `_s` in `group.field` or facet `field` unless necessary for the analysis.
+3.  **Dimension vs. Measure**:
+    *   `analysis_dimension`: The primary categorical field the user wants to group by (e.g., `company_name`, `route_branch`). This is the `group by` field.
+    *   `analysis_measure`: The metric to aggregate (e.g., `sum(total_deal_value_in_million)`) or the method of counting (`count`).
+    *   `sort_field_for_examples`: The raw field used to find the "best" example. If `analysis_measure` is `sum(field)`, this should be `field`. If `analysis_measure` is `count`, this should be a relevant field like `date`.
+4.  **Crucial Sorting Rules**:
+    *   For `group.sort`: If `analysis_measure` involves a function on a field (e.g., `sum(total_deal_value_in_million)`), you MUST use the full function: `group.sort: 'sum(total_deal_value_in_million) desc'`.
+    *   If `analysis_measure` is 'count', you MUST OMIT the `group.sort` parameter entirely.
+    *   For sorting, NEVER use 'date_year'; use 'date' instead.
+5.  **Output Format**: Your final output must be a single, raw JSON object. Do not add comments or markdown formatting.
+
+---
+### FIELD DEFINITIONS (Your Source of Truth)
+
+{formatted_field_info}
+---
+### CHAT HISTORY
+{formatted_history}
+---
+### EXAMPLES
+
+**User Query 1:** "What are the top 5 companies by total deal value in 2023?"
+**Correct JSON Output 1:**
+```json
+{{
+  "analysis_dimension": "company_name",
+  "analysis_measure": "sum(total_deal_value_in_million)",
+  "sort_field_for_examples": "total_deal_value_in_million",
+  "query_filter": "date_year:2023 AND total_deal_value_in_million:[0 TO *]",
+  "quantitative_request": {{
+    "json.facet": {{
+      "companies_by_deal_value": {{
+        "type": "terms",
+        "field": "company_name",
+        "limit": 5,
+        "sort": "total_value desc",
+        "facet": {{
+          "total_value": "sum(total_deal_value_in_million)"
+        }}
+      }}
+    }}
+  }},
+  "qualitative_request": {{
+    "group": true,
+    "group.field": "company_name",
+    "group.limit": 1,
+    "group.sort": "sum(total_deal_value_in_million) desc",
+    "sort": "total_deal_value_in_million desc"
+  }}
+}}
+```
+
+**User Query 2:** "What are the most common news types for infections this year?"
+**Correct JSON Output 2:**
+```json
+{{
+  "analysis_dimension": "news_type",
+  "analysis_measure": "count",
+  "sort_field_for_examples": "date",
+  "query_filter": "therapeutic_category_s:infections AND date_year:{datetime.datetime.now().year}",
+  "quantitative_request": {{
+    "json.facet": {{
+      "news_by_type": {{
+        "type": "terms",
+        "field": "news_type",
+        "limit": 10,
+        "sort": "count desc"
+      }}
+    }}
+  }},
+  "qualitative_request": {{
+    "group": true,
+    "group.field": "news_type",
+    "group.limit": 1,
+    "sort": "date desc"
+  }}
+}}
+```
+---
+### YOUR TASK
+
+Convert the following user query into a single, raw JSON "Analysis Plan" object, strictly following all rules and considering the chat history.
+
+**Current User Query:** `{natural_language_query}`
+"""
+
+def get_synthesis_report_prompt(query, quantitative_data, qualitative_data, plan):
+    """
+    Generates the prompt for synthesizing a final report from the query results.
+    """
+    qualitative_prompt_str = ""
+    dimension = plan.get('analysis_dimension', 'N/A')
+    if qualitative_data and dimension in qualitative_data:
+        for group in qualitative_data.get(dimension, {}).get('groups', []):
+            group_value = group.get('groupValue', 'N/A')
+            if group.get('doclist', {}).get('docs'):
+                doc = group.get('doclist', {}).get('docs', [{}])[0]
+                title = doc.get('abstract', ['No Title'])
+                content_list = doc.get('content', [])
+                content_snip = (' '.join(content_list[0].split()[:40]) + '...') if content_list else 'No content available.'
+                metric_val_raw = doc.get(plan.get('sort_field_for_examples'), 'N/A')
+                metric_val = metric_val_raw[0] if isinstance(metric_val_raw, list) else metric_val_raw
+
+                qualitative_prompt_str += f"- **For category `{group_value}`:**\n"
+                qualitative_prompt_str += f"  - **Top Example Title:** {title}\n"
+                qualitative_prompt_str += f"  - **Metric Value:** {metric_val}\n"
+                qualitative_prompt_str += f"  - **Content Snippet:** {content_snip}\n\n"
+
+    return f"""
+You are a top-tier business intelligence analyst. Your task is to write an insightful, data-driven report for an executive. You must synthesize quantitative data (the 'what') with qualitative examples (the 'why') to tell a complete story.
+
+---
+### AVAILABLE INFORMATION
+
+**1. The User's Core Question:**
+\"{query}\"
+
+**2. Quantitative Data (The 'What'):**
+This data shows the high-level aggregates.
+```json
+{json.dumps(quantitative_data, indent=2)}
+```
+
+**3. Qualitative Data (The 'Why'):
+These are the single most significant documents driving the numbers for each category.
+{qualitative_prompt_str}
+
+---
+### REPORTING INSTRUCTIONS
+
+Your report must be in clean, professional Markdown and follow this structure precisely.
+
+**Report Structure:**
+
+`## Executive Summary`
+- A 1-2 sentence, top-line answer to the user's question based on the quantitative data.
+
+`### Key Findings`
+- Use bullet points to highlight the main figures from the quantitative data. Interpret the numbers.
+
+`### Key Drivers & Illustrative Examples`
+- **This is the most important section.** Explain the "so what?" behind the numbers.
+- Use the qualitative examples to explain *why* a category is high or low. Reference the top example document for each main category.
+
+`### Deeper Dive: Suggested Follow-up Analyses`
+- Propose 2-3 logical next questions based on your analysis to uncover deeper trends.
+
+---
+**Generate the full report now, paying close attention to all formatting and spacing rules.**
+"""
+
+def get_visualization_code_prompt(query_context, facet_data):
+    """
+    Generates the prompt for creating Python visualization code.
+    """
+    return f"""
+You are a Python Data Visualization expert specializing in Matplotlib and Seaborn.
+Your task is to generate robust, error-free Python code to create a single, insightful visualization based on the user's query and the provided Solr facet data.
+
+**User's Analytical Goal:**
+\"{query_context}\"
+
+**Aggregated Data (from Solr Facets):**
+```json
+{json.dumps(facet_data, indent=2)}
+```
+
+---
+### **CRITICAL INSTRUCTIONS: CODE GENERATION RULES**
+You MUST follow these rules to avoid errors.
+
+**1. Identify the Data Structure FIRST:**
+Before writing any code, analyze the `facet_data` JSON to determine its structure. There are three common patterns. Choose the correct template below.
+
+   *   **Pattern A: Simple `terms` Facet.** The JSON has ONE main key (besides "count") which contains a list of "buckets". Each bucket has a "val" and a "count". Use this for standard bar charts.
+   *   **Pattern B: Multiple `query` Facets.** The JSON has MULTIPLE keys (besides "count"), and each key is an object containing metrics like "count" or "sum(...)". Use this for comparing a few distinct items (e.g., "oral vs injection").
+   *   **Pattern C: Nested `terms` Facet.** The JSON has one main key with a list of "buckets", but inside EACH bucket, there are nested metric objects. This is used for grouped comparisons (e.g., "compare 2024 vs 2025 across categories"). This almost always requires `pandas`.
+
+**2. Use the Correct Parsing Template:**
+
+---
+**TEMPLATE FOR PATTERN A (Simple Bar Chart from `terms` facet):**
+```python
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+
+plt.style.use('seaborn-v0_8-whitegrid')
+fig, ax = plt.subplots(figsize=(12, 8))
+
+# Dynamically find the main facet key (the one with 'buckets')
+facet_key = None
+for key, value in facet_data.items():
+    if isinstance(value, dict) and 'buckets' in value:
+        facet_key = key
+        break
+
+if facet_key:
+    buckets = facet_data[facet_key].get('buckets', [])
+    # Check if buckets contain data
+    if buckets:
+        df = pd.DataFrame(buckets)
+        # Check for a nested metric or use 'count'
+        if 'total_deal_value' in df.columns and pd.api.types.is_dict_like(df['total_deal_value'].iloc):
+             # Example for nested sum metric
+             df['value'] = df['total_deal_value'].apply(lambda x: x.get('sum', 0))
+             y_axis_label = 'Sum of Total Deal Value'
+        else:
+             df.rename(columns={{'count': 'value'}}, inplace=True)
+             y_axis_label = 'Count'
+
+        sns.barplot(data=df, x='val', y='value', ax=ax, palette='viridis')
+        ax.set_xlabel('Category')
+        ax.set_ylabel(y_axis_label)
+    else:
+        ax.text(0.5, 0.5, 'No data in buckets to plot.', ha='center')
+
+
+ax.set_title('Your Insightful Title Here')
+# Correct way to rotate labels to prevent errors
+plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+plt.tight_layout()
+```
+---
+**TEMPLATE FOR PATTERN B (Comparison Bar Chart from `query` facets):**
+```python
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+
+plt.style.use('seaborn-v0_8-whitegrid')
+fig, ax = plt.subplots(figsize=(10, 6))
+
+labels = []
+values = []
+# Iterate through top-level keys, skipping the 'count'
+for key, data_dict in facet_data.items():
+    if key == 'count' or not isinstance(data_dict, dict):
+        continue
+    # Extract the label (e.g., 'oral_deals' -> 'Oral')
+    label = key.replace('_deals', '').replace('_', ' ').title()
+    # Find the metric value, which is NOT 'count'
+    metric_value = 0
+    for sub_key, sub_value in data_dict.items():
+        if sub_key != 'count':
+            metric_value = sub_value
+            break # Found the metric
+    labels.append(label)
+    values.append(metric_value)
+
+if labels:
+    sns.barplot(x=labels, y=values, ax=ax, palette='mako')
+    ax.set_ylabel('Total Deal Value') # Or other metric name
+    ax.set_xlabel('Category')
+else:
+    ax.text(0.5, 0.5, 'No query facet data to plot.', ha='center')
+
+
+ax.set_title('Your Insightful Title Here')
+plt.tight_layout()
+```
+---
+**TEMPLATE FOR PATTERN C (Grouped Bar Chart from nested `terms` facet):**
+```python
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+
+plt.style.use('seaborn-v0_8-whitegrid')
+fig, ax = plt.subplots(figsize=(14, 8))
+
+# Find the key that has the buckets
+facet_key = None
+for key, value in facet_data.items():
+    if isinstance(value, dict) and 'buckets' in value:
+        facet_key = key
+        break
+
+if facet_key and facet_data[facet_key].get('buckets'):
+    # This list comprehension is robust for parsing nested metrics
+    plot_data = []
+    for bucket in facet_data[facet_key]['buckets']:
+        category = bucket['val']
+        # Find all nested metrics (e.g., total_deal_value_2025)
+        for sub_key, sub_value in bucket.items():
+            if isinstance(sub_value, dict) and 'sum' in sub_value:
+                # Extracts year from 'total_deal_value_2025' -> '2025'
+                year = sub_key.split('_')[-1]
+                value = sub_value['sum']
+                plot_data.append({{'Category': category, 'Year': year, 'Value': value}})
+
+    if plot_data:
+        df = pd.DataFrame(plot_data)
+        sns.barplot(data=df, x='Category', y='Value', hue='Year', ax=ax)
+        ax.set_ylabel('Total Deal Value')
+        ax.set_xlabel('Business Model')
+        # Correct way to rotate labels to prevent errors
+        plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+    else:
+        ax.text(0.5, 0.5, 'No nested data found to plot.', ha='center')
+else:
+    ax.text(0.5, 0.5, 'No data in buckets to plot.', ha='center')
+
+ax.set_title('Your Insightful Title Here')
+plt.tight_layout()
+```
+---
+**3. Final Code Generation:**
+- **DO NOT** include `plt.show()`.
+- **DO** set a dynamic and descriptive `ax.set_title()`, `ax.set_xlabel()`, and `ax.set_ylabel()`.
+- **DO NOT** wrap the code in ```python ... ```. Output only the raw Python code.
+- Adapt the chosen template to the specific keys and metrics in the provided `facet_data`.
+
+**Your Task:**
+Now, generate the Python code.
+"""

solr_metadata.py

@@ -0,0 +1,152 @@
+"""
+Stores the detailed metadata for the Solr index fields.
+
+This information is crucial for the LLM to understand the data schema,
+enabling it to construct accurate and efficient Solr queries. Separating it
+into its own module keeps the main application logic cleaner.
+"""
+
+field_metadata = [
+    {
+        "field_name": "business_model",
+        "type": "string (categorical)",
+        "example_values": ["pharma/bio", "drug delivery", "pharma services"],
+        "definition": "The primary business category of the company involved in the news. Use for filtering by high-level industry segments."
+    },
+    {
+        "field_name": "news_type",
+        "type": "string (categorical)",
+        "example_values": ["product news", "financial news", "regulatory news"],
+        "definition": "The category of the news article itself (e.g., financial, regulatory, acquisition). Use for filtering by the type of event being reported."
+    },
+    {
+        "field_name": "event_type",
+        "type": "string (categorical)",
+        "example_values": ["phase 2", "phase 1", "pre clinical", "marketed"],
+        "definition": "The clinical or developmental stage of a product or event discussed in the article. Essential for queries about clinical trial phases."
+    },
+    {
+        "field_name": "source",
+        "type": "string (categorical)",
+        "example_values": ["Press Release", "PR Newswire", "Business Wire"],
+        "definition": "The original source of the news article, such as a newswire or official report."
+    },
+    {
+        "field_name": "company_name",
+        "type": "string (exact match, for faceting)",
+        "example_values": ["pfizer inc.", "astrazeneca plc", "roche"],
+        "definition": "The canonical, standardized name of a company. **Crucially, you MUST use this field for `terms` faceting** to group results by a unique company. Do NOT use this for searching."
+    },
+    {
+        "field_name": "company_name_s",
+        "type": "string (multi-valued, for searching)",
+        "example_values": ["pfizer inc.", "roche", "f. hoffmann-la roche ag", "nih"],
+        "definition": "A field containing all known names and synonyms for a company. **You MUST use this field for all `query` parameter searches involving a company name** to ensure comprehensive results. Do NOT use for `terms` faceting."
+    },
+    {
+        "field_name": "territory_hq_s",
+        "type": "string (multi-valued, hierarchical)",
+        "example_values": ["united states of america", "europe", "europe western"],
+        "definition": "The geographic location (country and continent) of a company's headquarters. It is hierarchical. Use for filtering by location."
+    },
+    {
+        "field_name": "therapeutic_category",
+        "type": "string (specific)",
+        "example_values": ["cancer, other", "cancer, nsclc metastatic", "alzheimer's"],
+        "definition": "The specific disease or therapeutic area being targeted. Use for very specific disease queries."
+    },
+    {
+        "field_name": "therapeutic_category_s",
+        "type": "string (multi-valued, for searching)",
+        "example_values": ["cancer", "oncology", "infections", "cns"],
+        "definition": "Broader, multi-valued therapeutic categories and their synonyms. **Use this field for broad category searches** in the `query` parameter."
+    },
+    {
+        "field_name": "compound_name",
+        "type": "string (exact match, for faceting)",
+        "example_values": ["opdivo injection solution", "keytruda injection solution"],
+        "definition": "The specific, full trade name of a drug. **Use this field for `terms` faceting** on compounds."
+    },
+    {
+        "field_name": "compound_name_s",
+        "type": "string (multi-valued, for searching)",
+        "example_values": ["nivolumab injection solution", "opdivo injection solution", "ono-4538 injection solution"],
+        "definition": "A field with all known trade names and synonyms for a drug. **Use this field for all `query` parameter searches** involving a compound name."
+    },
+    {
+        "field_name": "molecule_name",
+        "type": "string (exact match, for faceting)",
+        "example_values": ["cannabidiol", "paclitaxel", "pembrolizumab"],
+        "definition": "The generic, non-proprietary name of the active molecule. **Use this field for `terms` faceting** on molecules."
+    },
+    {
+        "field_name": "molecule_name_s",
+        "type": "string (multi-valued, for searching)",
+        "example_values": ["cbd", "s1-220", "a1002n5s"],
+        "definition": "A field with all known generic names and synonyms for a molecule. **Use this field for all `query` parameter searches** involving a molecule name."
+    },
+    {
+        "field_name": "highest_phase",
+        "type": "string (categorical)",
+        "example_values": ["marketed", "phase 2", "phase 1"],
+        "definition": "The highest stage of development a drug has ever reached."
+    },
+    {
+        "field_name": "drug_delivery_branch_s",
+        "type": "string (multi-valued, for searching)",
+        "example_values": ["injection", "parenteral", "oral", "injection, other", "oral, other"],
+        "definition": "The method of drug administration. **Use this for `query` parameter searches about route of administration** as it contains broader, search-friendly terms."
+    },
+    {
+        "field_name": "drug_delivery_branch",
+        "type": "string (categorical, specific, for faceting)",
+        "example_values": ["injection, other", "prefilled syringes", "np liposome", "oral enteric/delayed release"],
+        "definition": "The most specific category of drug delivery technology. **Use this field for `terms` faceting** on specific delivery technologies."
+    },
+    {
+        "field_name": "route_branch",
+        "type": "string (categorical)",
+        "example_values": ["injection", "oral", "topical", "inhalation"],
+        "definition": "The primary route of drug administration. Good for faceting on exact routes."
+    },
+    {
+        "field_name": "molecule_api_group",
+        "type": "string (categorical)",
+        "example_values": ["small molecules", "biologics", "nucleic acids"],
+        "definition": "High-level classification of the drug's molecular type."
+    },
+    {
+        "field_name": "content",
+        "type": "text (full-text search)",
+        "example_values": ["The largest study to date...", "balstilimab..."],
+        "definition": "The full text content of the news article. Use for keyword searches on topics not covered by other specific fields."
+    },
+    {
+        "field_name": "date",
+        "type": "date",
+        "example_values": ["2020-10-22T00:00:00Z"],
+        "definition": "The full publication date and time in ISO 8601 format. Use for precise date range queries."
+    },
+    {
+        "field_name": "date_year",
+        "type": "number (year)",
+        "example_values": [2020, 2021, 2022],
+        "definition": "The 4-digit year of publication. **Use this for queries involving whole years** (e.g., 'in 2023', 'last year', 'since 2020')."
+    },
+    {
+        "field_name": "total_deal_value_in_million",
+        "type": "number (metric)",
+        "example_values": [50, 120.5, 176.157, 1000],
+        "definition": "The total value of a financial deal, in millions of USD. This is the primary numeric field for financial aggregations (sum, avg, etc.). To use this, you must also filter for news that has a deal value, e.g., 'total_deal_value_in_million:[0 TO *]'."
+    }
+]
+
+def format_metadata_for_prompt():
+    """Formats the field metadata into a string for the LLM prompt."""
+    formatted_string = ""
+    for field in field_metadata:
+        formatted_string += f"- **{field['field_name']}**\n"
+        formatted_string += f"  - **Type**: {field['type']}\n"
+        formatted_string += f"  - **Definition**: {field['definition']}\n"
+        formatted_string += f"  - **Examples**: {', '.join(map(str, field['example_values']))}\n\n"
+    return formatted_string

ui.py

@@ -0,0 +1,175 @@
+"""
+Defines the Gradio user interface and manages the application's state
+and event handling.
+
+This module is responsible for the presentation layer of the application.
+It creates the interactive components and orchestrates the analysis workflow
+by calling functions from the data_processing module.
+"""
+
+
+import gradio as gr
+import json
+import concurrent.futures
+from data_processing import (
+    llm_generate_analysis_plan_with_history,
+    execute_quantitative_query,
+    execute_qualitative_query,
+    llm_synthesize_enriched_report_stream,
+    llm_generate_visualization_code,
+    execute_viz_code_and_get_path,
+    parse_suggestions_from_report
+)
+
+def create_ui(llm_model, solr_client):
+    """
+    Builds the Gradio UI and wires up all the event handlers.
+
+    Args:
+        llm_model: The initialized Google Gemini model client.
+        solr_client: The initialized pysolr client.
+    """
+    with gr.Blocks(theme=gr.themes.Soft(), css="footer {display: none !important}") as demo:
+        state = gr.State()
+
+        with gr.Row():
+            with gr.Column(scale=4):
+                gr.Markdown("# PharmaCircle AI Data Analyst")
+            with gr.Column(scale=1):
+                clear_button = gr.Button("🔄 Start New Analysis", variant="primary")
+
+        gr.Markdown("Ask a question to begin your analysis. I will generate an analysis plan, retrieve quantitative and qualitative data, create a visualization, and write an enriched report.")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                chatbot = gr.Chatbot(label="Analysis Chat Log", height=700, show_copy_button=True)
+                msg_textbox = gr.Textbox(placeholder="Ask a question, e.g., 'Show me the top 5 companies by total deal value in 2023'", label="Your Question", interactive=True)
+
+            with gr.Column(scale=2):
+                with gr.Accordion("Generated Analysis Plan", open=False):
+                    plan_display = gr.Markdown("Plan will appear here...", visible=True)
+                with gr.Accordion("Retrieved Quantitative Data", open=False):
+                    quantitative_data_display = gr.Markdown("Aggregate data will appear here...", visible=False)
+                with gr.Accordion("Retrieved Qualitative Data (Examples)", open=False):
+                    qualitative_data_display = gr.Markdown("Example data will appear here...", visible=False)
+                plot_display = gr.Image(label="Visualization", type="filepath", visible=False)
+                report_display = gr.Markdown("Report will be streamed here...", visible=False)
+
+        def process_analysis_flow(user_input, history, state):
+            """
+            Manages the conversation and yields UI updates.
+            """
+            if state is None:
+                state = {'query_count': 0, 'last_suggestions': []}
+            if history is None:
+                history = []
+
+            yield (history, state, gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=None, visible=False))
+
+            query_context = user_input.strip()
+            if not query_context:
+                history.append((user_input, "Please enter a question to analyze."))
+                yield (history, state, None, None, None, None, None)
+                return
+
+            history.append((user_input, f"Analyzing: '{query_context}'\n\n*Generating analysis plan...*"))
+            yield (history, state, None, None, None, None, None)
+
+            analysis_plan = llm_generate_analysis_plan_with_history(llm_model, query_context, history)
+            if not analysis_plan:
+                history.append((None, "I'm sorry, I couldn't generate a valid analysis plan. Please try rephrasing."))
+                yield (history, state, None, None, None, None, None)
+                return
+
+            history.append((None, "✅ Analysis plan generated!"))
+            plan_summary = f"""
+*   **Analysis Dimension:** `{analysis_plan.get('analysis_dimension')}`
+*   **Analysis Measure:** `{analysis_plan.get('analysis_measure')}`
+*   **Query Filter:** `{analysis_plan.get('query_filter')}`
+"""
+            history.append((None, plan_summary))
+            formatted_plan = f"**Full Analysis Plan:**\n```json\n{json.dumps(analysis_plan, indent=2)}\n```"
+            yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
+
+            history.append((None, "*Executing queries for aggregates and examples...*"))
+            yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
+
+            aggregate_data = None
+            example_data = None
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                future_agg = executor.submit(execute_quantitative_query, solr_client, analysis_plan)
+                future_ex = executor.submit(execute_qualitative_query, solr_client, analysis_plan)
+                aggregate_data = future_agg.result()
+                example_data = future_ex.result()
+
+            if not aggregate_data or aggregate_data.get('count', 0) == 0:
+                history.append((None, "No data was found for your query. Please try a different question."))
+                yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), None, None)
+                return
+
+            formatted_agg_data = f"**Quantitative (Aggregate) Data:**\n```json\n{json.dumps(aggregate_data, indent=2)}\n```"
+            formatted_qual_data = f"**Qualitative (Example) Data:**\n```json\n{json.dumps(example_data, indent=2)}\n```"
+            qual_data_display_update = gr.update(value=formatted_qual_data, visible=True)
+            yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
+
+            history.append((None, "✅ Data retrieved. Generating visualization and final report..."))
+            yield (history, state, None, None, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
+
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                viz_future = executor.submit(llm_generate_visualization_code, llm_model, query_context, aggregate_data)
+
+                report_text = ""
+                stream_history = history[:]
+                for chunk in llm_synthesize_enriched_report_stream(llm_model, query_context, aggregate_data, example_data, analysis_plan):
+                    report_text += chunk
+                    yield (stream_history, state, None, gr.update(value=report_text, visible=True), gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
+
+                history.append((None, report_text))
+
+                viz_code = viz_future.result()
+                plot_path = execute_viz_code_and_get_path(viz_code, aggregate_data)
+                output_plot = gr.update(value=plot_path, visible=True) if plot_path else gr.update(visible=False)
+                if not plot_path:
+                    history.append((None, "*I was unable to generate a plot for this data.*\n"))
+
+                yield (history, state, output_plot, report_text, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
+
+            state['query_count'] += 1
+            state['last_suggestions'] = parse_suggestions_from_report(report_text)
+            next_prompt = "Analysis complete. What would you like to explore next?"
+            history.append((None, next_prompt))
+            yield (history, state, output_plot, report_text, gr.update(value=formatted_plan, visible=True), gr.update(value=formatted_agg_data, visible=True), qual_data_display_update)
+
+        def reset_all():
+            """Resets the entire UI for a new analysis session."""
+            return (
+                [],
+                None,
+                "",
+                gr.update(value=None, visible=False),
+                gr.update(value=None, visible=False),
+                gr.update(value=None, visible=False),
+                gr.update(value=None, visible=False),
+                gr.update(value=None, visible=False)
+            )
+
+        msg_textbox.submit(
+            fn=process_analysis_flow,
+            inputs=[msg_textbox, chatbot, state],
+            outputs=[chatbot, state, plot_display, report_display, plan_display, quantitative_data_display, qualitative_data_display],
+        ).then(
+            lambda: gr.update(value=""),
+            None,
+            [msg_textbox],
+            queue=False,
+        )
+
+        clear_button.click(
+            fn=reset_all,
+            inputs=None,
+            outputs=[chatbot, state, msg_textbox, plot_display, report_display, plan_display, quantitative_data_display, qualitative_data_display],
+            queue=False
+        )
+
+    return demo
+