Ported Gradio meta-prompt to Streamlit.

app/streamlit_meta_prompt.py

@@ -0,0 +1,613 @@
+import streamlit as st
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+from langchain_core.language_models import BaseLanguageModel
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI
+from meta_prompt import *
+from pythonjsonlogger import jsonlogger
+from app.config import MetaPromptConfig, RoleMessage
+from confz import BaseConfig, CLArgSource, EnvSource, FileSource
+import io
+
+def prompt_templates_confz2langchain(
+    prompt_templates: Dict[str, Dict[str, List[RoleMessage]]]
+) -> Dict[str, ChatPromptTemplate]:
+    return {
+        node: ChatPromptTemplate.from_messages(
+            [
+                (role_message.role, role_message.message)
+                for role_message in role_messages
+            ]
+        )
+        for node, role_messages in prompt_templates.items()
+    }
+
+class LLMModelFactory:
+    _instance = None
+
+    def __new__(cls):
+        if not cls._instance:
+            cls._instance = super(LLMModelFactory, cls).__new__(cls)
+        return cls._instance
+
+    def create(self, model_type: str, **kwargs) -> BaseLanguageModel:
+        model_class = globals()[model_type]
+        return model_class(**kwargs)
+
+def chat_log_2_chatbot_list(chat_log: str) -> List[List[str]]:
+    chatbot_list = []
+    if chat_log is None or chat_log == '':
+        return chatbot_list
+    for line in chat_log.splitlines():
+        try:
+            json_line = json.loads(line)
+            if 'action' in json_line:
+                if json_line['action'] == 'invoke':
+                    chatbot_list.append([json_line['message'], None])
+                if json_line['action'] == 'response':
+                    chatbot_list.append([None, json_line['message']])
+        except json.decoder.JSONDecodeError as e:
+            print(f"Error decoding JSON log output: {e}")
+            print(line)
+        except KeyError as e:
+            print(f"Error accessing key in JSON log output: {e}")
+            print(line)
+    return chatbot_list
+
+active_model_tab = "Simple"
+
+def get_current_model(simple_model_name: str,
+                        advanced_model_name: str,
+                        expert_model_name: str,
+                        expert_model_config: Optional[Dict[str, Any]] = None) -> BaseLanguageModel:
+    model_mapping = {
+        "Simple": simple_model_name,
+        "Advanced": advanced_model_name,
+        "Expert": expert_model_name
+    }
+    
+    try:
+        model_name = model_mapping.get(active_model_tab, simple_model_name)
+        model = config.llms[model_name]
+        model_type = model.type
+        model_config = model.model_dump(exclude={'type'})
+    
+        if active_model_tab == "Expert" and expert_model_config:
+            model_config.update(expert_model_config)
+    
+        return LLMModelFactory().create(model_type, **model_config)
+    
+    except KeyError as e:
+        logging.error(f"Configuration key error: {e}")
+        raise ValueError(f"Invalid model name or configuration: {e}")
+    
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {e}")
+        raise RuntimeError(f"Failed to retrieve the model: {e}")
+
+def evaluate_system_message(system_message, user_message, simple_model,
+                            advanced_executor_model, expert_executor_model,
+                            expert_executor_model_temperature=0.1):
+    llm = get_current_model(simple_model, advanced_executor_model,
+                            expert_executor_model,
+                            {"temperature": expert_executor_model_temperature})
+    template = ChatPromptTemplate.from_messages([
+        ("system", "{system_message}"),
+        ("human", "{user_message}")
+    ])
+    try:
+        output = llm.invoke(template.format(
+            system_message=system_message, user_message=user_message))
+        return output.content if hasattr(output, 'content') else ""
+    except Exception as e:
+        raise st.error(f"Error: {e}")
+
+def generate_acceptance_criteria(user_message, expected_output,
+                                 simple_model, advanced_executor_model,
+                                 expert_prompt_acceptance_criteria_model,
+                                 expert_prompt_acceptance_criteria_temperature=0.1,
+                                 prompt_template_group: Optional[str] = None):
+    log_stream = io.StringIO()
+    logger = logging.getLogger(MetaPromptGraph.__name__) if config.verbose else None
+    log_handler = logging.StreamHandler(log_stream) if logger else None
+
+    if log_handler:
+        log_handler.setFormatter(
+            jsonlogger.JsonFormatter('%(asctime)s %(name)s %(levelname)s %(message)s')
+        )
+        logger.addHandler(log_handler)
+
+    llm = get_current_model(simple_model, advanced_executor_model,
+                            expert_prompt_acceptance_criteria_model,
+                            {"temperature": expert_prompt_acceptance_criteria_temperature})
+    if prompt_template_group is None:
+        prompt_template_group = 'default'
+    prompt_templates = prompt_templates_confz2langchain(
+        config.prompt_templates[prompt_template_group]
+    )
+    acceptance_criteria_graph = MetaPromptGraph(llms={
+        NODE_ACCEPTANCE_CRITERIA_DEVELOPER: llm
+    }, prompts=prompt_templates,
+    verbose=config.verbose, logger=logger)
+    state = AgentState(
+        user_message=user_message,
+        expected_output=expected_output
+    )
+    output_state = acceptance_criteria_graph.run_acceptance_criteria_graph(state)
+
+    if log_handler:
+        log_handler.close()
+        log_output = log_stream.getvalue()
+    else:
+        log_output = None
+    return output_state.get('acceptance_criteria', ""), chat_log_2_chatbot_list(log_output)
+
+def generate_initial_system_message(
+    user_message: str,
+    expected_output: str,
+    simple_model: str,
+    advanced_executor_model: str,
+    expert_prompt_initial_developer_model: str,
+    expert_prompt_initial_developer_temperature: float = 0.1,
+    prompt_template_group: Optional[str] = None
+) -> tuple:
+    log_stream = io.StringIO()
+    logger = logging.getLogger(MetaPromptGraph.__name__) if config.verbose else None
+    log_handler = logging.StreamHandler(log_stream) if logger else None
+
+    if log_handler:
+        log_handler.setFormatter(
+            jsonlogger.JsonFormatter('%(asctime)s %(name)s %(levelname)s %(message)s')
+        )
+        logger.addHandler(log_handler)
+
+    llm = get_current_model(
+        simple_model,
+        advanced_executor_model,
+        expert_prompt_initial_developer_model,
+        {"temperature": expert_prompt_initial_developer_temperature}
+    )
+
+    if prompt_template_group is None:
+        prompt_template_group = 'default'
+    prompt_templates = prompt_templates_confz2langchain(
+        config.prompt_templates[prompt_template_group]
+    )
+
+    initial_system_message_graph = MetaPromptGraph(
+        llms={NODE_PROMPT_INITIAL_DEVELOPER: llm},
+        prompts=prompt_templates,
+        verbose=config.verbose,
+        logger=logger
+    )
+
+    state = AgentState(
+        user_message=user_message,
+        expected_output=expected_output
+    )
+
+    output_state = initial_system_message_graph.run_prompt_initial_developer_graph(state)
+
+    if log_handler:
+        log_handler.close()
+        log_output = log_stream.getvalue()
+    else:
+        log_output = None
+
+    system_message = output_state.get('system_message', "")
+    return system_message, chat_log_2_chatbot_list(log_output)
+
+def process_message(
+    user_message: str, expected_output: str, acceptance_criteria: str,
+    initial_system_message: str, recursion_limit: int, max_output_age: int,
+    llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]],
+    prompt_template_group: Optional[str] = None,
+    aggressive_exploration: bool = False
+) -> tuple:
+    input_state = AgentState(
+        user_message=user_message,
+        expected_output=expected_output,
+        acceptance_criteria=acceptance_criteria,
+        system_message=initial_system_message,
+        max_output_age=max_output_age
+    )
+
+    log_stream = io.StringIO()
+    logger = logging.getLogger(MetaPromptGraph.__name__) if config.verbose else None
+    log_handler = logging.StreamHandler(log_stream) if logger else None
+    if log_handler:
+        log_handler.setFormatter(jsonlogger.JsonFormatter(
+            '%(asctime)s %(name)s %(levelname)s %(message)s'))
+        logger.addHandler(log_handler)
+
+    if prompt_template_group is None:
+        prompt_template_group = 'default'
+    prompt_templates = prompt_templates_confz2langchain(config.prompt_templates[prompt_template_group])
+    meta_prompt_graph = MetaPromptGraph(llms=llms, prompts=prompt_templates,
+                                        aggressive_exploration=aggressive_exploration,
+                                        verbose=config.verbose, logger=logger)
+    try:
+        output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
+    except Exception as e:
+        raise st.error(f"Error: {e}")           
+
+    if log_handler:
+        log_handler.close()
+        log_output = log_stream.getvalue()
+    else:
+        log_output = None
+
+    system_message = output_state.get(
+        'best_system_message', "Error: The output state does not contain a valid 'best_system_message'")
+    output = output_state.get(
+        'best_output', "Error: The output state does not contain a valid 'best_output'")
+    analysis = output_state.get(
+        'analysis', "Error: The output state does not contain a valid 'analysis'")
+    acceptance_criteria = output_state.get(
+        'acceptance_criteria', "Error: The output state does not contain a valid 'acceptance_criteria'")
+
+    return (system_message, output, analysis, acceptance_criteria, chat_log_2_chatbot_list(log_output))
+
+def initialize_llm(model_name: str, model_config: Optional[Dict[str, Any]] = None) -> Any:
+    try:
+        llm_config = config.llms[model_name]
+        model_type = llm_config.type
+        dumped_config = llm_config.model_dump(exclude={'type'})
+
+        if model_config:
+            dumped_config.update(model_config)
+
+        return LLMModelFactory().create(model_type, **dumped_config)
+    except KeyError:
+        raise KeyError(f"No configuration exists for the model name: {model_name}")
+    except NotImplementedError:
+        raise NotImplementedError(
+            f"Unrecognized type configured for the language model: {model_type}"
+        )
+
+def process_message_with_single_llm(
+    user_message: str, expected_output: str, acceptance_criteria: str,
+    initial_system_message: str, recursion_limit: int, max_output_age: int,
+    model_name: str, prompt_template_group: Optional[str] = None,
+    aggressive_exploration: bool = False
+) -> tuple:
+    llm = initialize_llm(model_name)
+    return process_message(
+        user_message, expected_output, acceptance_criteria, initial_system_message,
+        recursion_limit, max_output_age, llm, prompt_template_group, aggressive_exploration
+    )
+
+def process_message_with_2_llms(
+    user_message: str, expected_output: str, acceptance_criteria: str,
+    initial_system_message: str, recursion_limit: int, max_output_age: int,
+    optimizer_model_name: str, executor_model_name: str,
+    prompt_template_group: Optional[str] = None,
+    aggressive_exploration: bool = False
+) -> tuple:
+    optimizer_model = initialize_llm(optimizer_model_name)
+    executor_model = initialize_llm(executor_model_name)
+    llms = {
+        NODE_ACCEPTANCE_CRITERIA_DEVELOPER: optimizer_model,
+        NODE_PROMPT_INITIAL_DEVELOPER: optimizer_model,
+        NODE_PROMPT_DEVELOPER: optimizer_model,
+        NODE_PROMPT_EXECUTOR: executor_model,
+        NODE_OUTPUT_HISTORY_ANALYZER: optimizer_model,
+        NODE_PROMPT_ANALYZER: optimizer_model,
+        NODE_PROMPT_SUGGESTER: optimizer_model
+    }
+    return process_message(
+        user_message, expected_output, acceptance_criteria,
+        initial_system_message, recursion_limit, max_output_age, llms,
+        prompt_template_group, aggressive_exploration
+    )
+
+def process_message_with_expert_llms(
+    user_message: str, expected_output: str, acceptance_criteria: str,
+    initial_system_message: str, recursion_limit: int, max_output_age: int,
+    initial_developer_model_name: str, initial_developer_temperature: float,
+    acceptance_criteria_model_name: str, acceptance_criteria_temperature: float,
+    developer_model_name: str, developer_temperature: float,
+    executor_model_name: str, executor_temperature: float,
+    output_history_analyzer_model_name: str, output_history_analyzer_temperature: float,
+    analyzer_model_name: str, analyzer_temperature: float,
+    suggester_model_name: str, suggester_temperature: float,
+    prompt_template_group: Optional[str] = None, aggressive_exploration: bool = False
+) -> tuple:
+    llms = {
+        NODE_PROMPT_INITIAL_DEVELOPER: initialize_llm(
+            initial_developer_model_name, {"temperature": initial_developer_temperature}
+        ),
+        NODE_ACCEPTANCE_CRITERIA_DEVELOPER: initialize_llm(
+            acceptance_criteria_model_name, {"temperature": acceptance_criteria_temperature}
+        ),
+        NODE_PROMPT_DEVELOPER: initialize_llm(
+            developer_model_name, {"temperature": developer_temperature}
+        ),
+        NODE_PROMPT_EXECUTOR: initialize_llm(
+            executor_model_name, {"temperature": executor_temperature}
+        ),
+        NODE_OUTPUT_HISTORY_ANALYZER: initialize_llm(
+            output_history_analyzer_model_name,
+            {"temperature": output_history_analyzer_temperature}
+        ),
+        NODE_PROMPT_ANALYZER: initialize_llm(
+            analyzer_model_name, {"temperature": analyzer_temperature}
+        ),
+        NODE_PROMPT_SUGGESTER: initialize_llm(
+            suggester_model_name, {"temperature": suggester_temperature}
+        )
+    }
+    return process_message(
+        user_message,
+        expected_output,
+        acceptance_criteria,
+        initial_system_message,
+        recursion_limit,
+        max_output_age,
+        llms,
+        prompt_template_group,
+        aggressive_exploration
+    )
+
+class FileConfig(BaseConfig):
+    config_file: str = 'config.yml'  # default path
+
+pre_config_sources = [
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+pre_config = FileConfig(config_sources=pre_config_sources)
+
+config_sources = [
+    FileSource(file=pre_config.config_file, optional=True),
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+
+config = MetaPromptConfig(config_sources=config_sources)
+
+# Streamlit UI
+st.title("Meta Prompt")
+st.markdown("A tool for generating and analyzing natural language prompts using multiple language models.")
+
+with st.sidebar:
+    st.header("Model Settings")
+    model_tab = st.selectbox("Select Model Type", ["Simple", "Advanced", "Expert"], key="model_tab")
+
+    if model_tab == "Simple":
+        simple_model_name_input = st.selectbox(
+            "Model Name",
+            config.llms.keys(),
+            index=0,
+        )
+    elif model_tab == "Advanced":
+        advanced_optimizer_model_name_input = st.selectbox(
+            "Optimizer Model Name",
+            config.llms.keys(),
+            index=0,
+        )
+        advanced_executor_model_name_input = st.selectbox(
+            "Executor Model Name",
+            config.llms.keys(),
+            index=1,
+        )
+    else:  # Expert
+        expert_prompt_initial_developer_model_name_input = st.selectbox(
+            "Initial Developer Model Name",
+            config.llms.keys(),
+            index=0,
+        )
+        expert_prompt_initial_developer_temperature_input = st.slider(
+            "Initial Developer Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_acceptance_criteria_model_name_input = st.selectbox(
+            "Acceptance Criteria Model Name",
+            config.llms.keys(),
+            index=0,
+        )
+        expert_prompt_acceptance_criteria_temperature_input = st.slider(
+            "Acceptance Criteria Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_developer_model_name_input = st.selectbox(
+            "Developer Model Name", config.llms.keys(), index=0
+        )
+        expert_prompt_developer_temperature_input = st.slider(
+            "Developer Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_executor_model_name_input = st.selectbox(
+            "Executor Model Name", config.llms.keys(), index=1
+        )
+        expert_prompt_executor_temperature_input = st.slider(
+            "Executor Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_output_history_analyzer_model_name_input = st.selectbox(
+            "Output History Analyzer Model Name",
+            config.llms.keys(),
+            index=0,
+        )
+        expert_prompt_output_history_analyzer_temperature_input = st.slider(
+            "Output History Analyzer Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_analyzer_model_name_input = st.selectbox(
+            "Analyzer Model Name", config.llms.keys(), index=0
+        )
+        expert_prompt_analyzer_temperature_input = st.slider(
+            "Analyzer Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+        expert_prompt_suggester_model_name_input = st.selectbox(
+            "Suggester Model Name", config.llms.keys(), index=0
+        )
+        expert_prompt_suggester_temperature_input = st.slider(
+            "Suggester Temperature", 0.0, 1.0, 0.1, 0.1
+        )
+
+    st.header("Prompt Template Settings")
+    prompt_template_group_input = st.selectbox(
+        "Prompt Template Group", config.prompt_templates.keys(), index=0
+    )
+
+    st.header("Advanced Settings")
+    recursion_limit_input = st.number_input("Recursion Limit", 1, 100, 16, 1)
+    max_output_age_input = st.number_input("Max Output Age", 1, 10, 2, 1)
+    aggressive_exploration_input = st.checkbox("Aggressive Exploration", False)
+
+if __name__ == "__main__":
+    # Initialize session state
+    if 'initial_system_message' not in st.session_state:
+        st.session_state.initial_system_message = ""
+    if 'initial_acceptance_criteria' not in st.session_state:
+        st.session_state.initial_acceptance_criteria = ""
+    if 'system_message' not in st.session_state:
+        st.session_state.system_message = ""
+    if 'output' not in st.session_state:  
+        st.session_state.output = ""
+    if 'analysis' not in st.session_state:
+        st.session_state.analysis = ""
+    if 'acceptance_criteria_output' not in st.session_state:
+        st.session_state.acceptance_criteria_output = ""
+    if 'chat_log' not in st.session_state:
+        st.session_state.chat_log = []
+
+    def copy_system_message():
+        if 'system_message_output' in st.session_state:
+            st.session_state.initial_system_message = st.session_state.system_message_output
+
+    def copy_acceptance_criteria():
+        if 'acceptance_criteria_output' in st.session_state:
+            st.session_state.initial_acceptance_criteria = st.session_state.acceptance_criteria_output
+
+    if active_model_tab == "Simple":
+        simple_model_name = simple_model_name_input
+        advanced_executor_model_name = None
+        expert_prompt_initial_developer_model_name = None
+        expert_prompt_acceptance_criteria_model_name = None
+        expert_prompt_developer_model_name = None
+        expert_prompt_executor_model_name = None
+        expert_prompt_output_history_analyzer_model_name = None
+        expert_prompt_analyzer_model_name = None
+        expert_prompt_suggester_model_name = None
+    elif active_model_tab == "Advanced":
+        simple_model_name = None
+        advanced_executor_model_name = advanced_executor_model_name_input
+        expert_prompt_initial_developer_model_name = None
+        expert_prompt_acceptance_criteria_model_name = None
+        expert_prompt_developer_model_name = None
+        expert_prompt_executor_model_name = None
+        expert_prompt_output_history_analyzer_model_name = None
+        expert_prompt_analyzer_model_name = None
+        expert_prompt_suggester_model_name = None
+    else:  # Expert
+        simple_model_name = None
+        advanced_executor_model_name = None
+        expert_prompt_initial_developer_model_name = (
+            expert_prompt_initial_developer_model_name_input
+        )
+        expert_prompt_acceptance_criteria_model_name = (
+            expert_prompt_acceptance_criteria_model_name_input
+        )
+        expert_prompt_developer_model_name = expert_prompt_developer_model_name_input
+        expert_prompt_executor_model_name = expert_prompt_executor_model_name_input
+        expert_prompt_output_history_analyzer_model_name = (
+            expert_prompt_output_history_analyzer_model_name_input
+        )
+        expert_prompt_analyzer_model_name = expert_prompt_analyzer_model_name_input
+        expert_prompt_suggester_model_name = expert_prompt_suggester_model_name_input
+
+    prompt_template_group = prompt_template_group_input
+    recursion_limit = recursion_limit_input
+    max_output_age = max_output_age_input
+    aggressive_exploration = aggressive_exploration_input
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        user_message = st.text_area("User Message", "").strip()
+        expected_output = st.text_area("Expected Output", "").strip()
+        initial_system_message = st.text_area("Initial System Message", st.session_state.initial_system_message).strip()
+        acceptance_criteria = st.text_area("Acceptance Criteria", st.session_state.initial_acceptance_criteria).strip()
+        
+        generate_button_clicked = st.button("Generate", type="primary")
+
+    with col2:
+        if generate_button_clicked:
+            try:
+                if active_model_tab == "Simple":
+                    system_message, output, analysis, acceptance_criteria, chat_log = process_message_with_single_llm(
+                        user_message,
+                        expected_output,
+                        acceptance_criteria,
+                        initial_system_message,
+                        recursion_limit,
+                        max_output_age,
+                        simple_model_name,
+                        prompt_template_group,
+                        aggressive_exploration,
+                    )
+                elif active_model_tab == "Advanced":
+                    system_message, output, analysis, acceptance_criteria, chat_log = process_message_with_2_llms(
+                        user_message,
+                        expected_output,
+                        acceptance_criteria,
+                        initial_system_message,
+                        recursion_limit,
+                        max_output_age,
+                        advanced_optimizer_model_name_input,
+                        advanced_executor_model_name_input,
+                        prompt_template_group,
+                        aggressive_exploration,
+                    )
+                else:  # Expert
+                    system_message, output, analysis, acceptance_criteria, chat_log = process_message_with_expert_llms(
+                        user_message,
+                        expected_output,
+                        acceptance_criteria,
+                        initial_system_message,
+                        recursion_limit,
+                        max_output_age,
+                        expert_prompt_initial_developer_model_name,
+                        expert_prompt_initial_developer_temperature_input,
+                        expert_prompt_acceptance_criteria_model_name,
+                        expert_prompt_acceptance_criteria_temperature_input,
+                        expert_prompt_developer_model_name,
+                        expert_prompt_developer_temperature_input,
+                        expert_prompt_executor_model_name,
+                        expert_prompt_executor_temperature_input,
+                        expert_prompt_output_history_analyzer_model_name,
+                        expert_prompt_output_history_analyzer_temperature_input,
+                        expert_prompt_analyzer_model_name,
+                        expert_prompt_analyzer_temperature_input,
+                        expert_prompt_suggester_model_name,
+                        expert_prompt_suggester_temperature_input,
+                        prompt_template_group,
+                        aggressive_exploration,
+                    )
+                
+                st.session_state.system_message_output = system_message
+                st.session_state.output = output
+                st.session_state.analysis = analysis
+                st.session_state.acceptance_criteria_output = acceptance_criteria
+                st.session_state.chat_log = chat_log
+
+            except Exception as e:
+                st.error(f"Error: {e}")
+                
+        st.text_area("System Message",
+                     key="system_message_output", height=100)
+        st.button("Copy System Message", key="copy_system_message",
+                  on_click=copy_system_message)
+        st.text_area("Output", st.session_state.output, height=100)
+        st.text_area("Analysis", st.session_state.analysis, height=100)
+        acceptance_criteria_output = st.text_area(
+            "Acceptance Criteria", key="acceptance_criteria_output", height=100)
+        st.button("Copy Acceptance Criteria", key="copy_acceptance_criteria",
+                  on_click=copy_acceptance_criteria)
+        st.json(st.session_state.chat_log)

guidelines/gradio.md

@@ -0,0 +1,29 @@
+# Gradio Development Guidelines
+
+## Best Practices
+
+- Use `gr.Blocks` to create a structured UI layout with rows, columns, tabs, and accordions.
+- Organize related UI elements into groups using `gr.Group` for better readability and maintainability.
+- Provide clear labels and instructions for user inputs and interactions using `gr.Markdown` and `gr.Textbox`.
+- Use `gr.Dropdown` to allow users to select from a predefined list of options.
+- Implement buttons with `gr.Button` and assign appropriate callbacks using the `click` event.
+- Use `gr.Examples` to provide sample inputs for users to quickly test the app's functionality.
+- Handle file uploads and downloads using `gr.FileUpload` and `gr.FileDownload`.
+- Display output using appropriate components like `gr.Textbox`, `gr.Chatbot`, `gr.Dataframe`, etc.
+- Implement flagging functionality to allow users to report issues or provide feedback.
+- Use `gr.Accordion` to hide detailed information that may not be necessary for all users.
+- Provide a clear and concise title for the app using the `title` parameter in `gr.Blocks`.
+- Use `gr.ClearButton` to allow users to reset input fields and start over.
+
+## Principles
+
+- Prioritize usability and user experience in the app's design and layout.
+- Ensure the app is responsive and works well on different screen sizes and devices.
+- Optimize performance by minimizing unnecessary computations and caching results when possible.
+- Follow PEP 8 style guidelines for Python code.
+- Document the app's purpose, usage instructions, and code to enhance maintainability.
+- Test the app thoroughly to identify and fix bugs, edge cases, and performance issues.
+- Consider accessibility and ensure the app can be used by people with different abilities.
+- Provide clear feedback to users about the app's status and results.
+- Allow users to customize the app's behavior through settings and options.
+- Design the app to be modular and extensible, allowing for future enhancements and new features.

guidelines/prompts.md

@@ -0,0 +1,348 @@
+# Prompts and Guidelines
+
+## Prompts
+
+### Prompt Initial Developer
+- **gpt**: 
+  - role: system
+    message: |
+      # Expert Prompt Engineer
+
+      You are an expert prompt engineer tasked with creating system messages for AI assistants.
+
+      ## Instructions
+
+      1. Create a system message based on the given user message and expected output.
+      2. Ensure the system message can handle similar user messages.
+      3. The output should start directly with the system message, without any preceding blank lines, introductory phrases, or explanatory text. Do not include extra lines at the beginning or end of the output.
+      4. Expected Output text should not appear in System Message as an example. But it's OK to use some similar text as an example instead.
+      5. In the System Message, do not use `Expected Output` to refer to the example you want to illustrate. Instead, directly describe the specific features you need.
+      6. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
+
+      ## Output
+
+      Provide only the system message, adhering to the above guidelines.
+
+### Prompt Developer
+- **gpt**: 
+  - role: system
+    message: |
+      # Expert Prompt Engineer
+
+      You are an expert prompt engineer tasked with updating system messages for AI assistants. You Update System Message according to Suggestions, to improve Output and match Expected Output more closely.
+
+      ## Instructions
+
+      1. Update the system message based on the given Suggestion, User Message, and Expected Output.
+      2. Ensure the updated system message can handle similar user messages.
+      3. Modify only the content mentioned in the Suggestion. Do not change the parts that are not related to the Suggestion.
+      4. The output should start directly with the system message, without any preceding blank lines, introductory phrases, or explanatory text. Do not include extra lines at the beginning or end of the output.
+      5. Avoiding the behavior should be explicitly requested (e.g. `Don't ...`) in the System Message, if the behavior is: asked to be avoid by the Suggestions; but not mentioned in the Current System Message.
+      6. Expected Output text should not appear in System Message as an example. But it's OK to use some similar text as an example instead.
+      7. In the System Message, do not use `Expected Output` to refer to the example you want to illustrate. Instead, directly describe the specific features you need.
+      8. Remove the Expected Output text or text highly similar to Expected Output from System Message, if it's present.
+      9. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
+
+      ## Output
+
+      Provide only the updated System Message, adhering to the above guidelines.
+
+### Prompt Executor
+- **gpt**: 
+  - role: system
+    message: "{system_message}"
+  - role: human
+    message: "{user_message}"
+
+### Output History Analyzer
+- **gpt**: 
+  - role: system
+    message: |
+      You are a text comparing program. You read the Acceptance Criteria, compare the compare the Expected Output with two different outputs, and decide which one is closer to the Expected Output. When comparing the outputs, ignore the differences which are acceptable or ignorable according to the Acceptance Criteria.
+
+      You output the following analysis according to the Acceptance Criteria:
+
+      * Your analysis in a Markdown list.
+      * Indicates an output ID that is closer to the Expected Output, in the following format:
+
+      ```
+      # Analysis
+
+      ...
+
+      # Output ID closer to Expected Output: [ID]
+      ```
+
+      You must choose one of the two outputs. If both outputs are exactly the same, output the following:
+
+      ```
+      # Analysis
+
+      ...
+
+      # Draw
+      ```
+
+### Prompt Analyzer
+- **gpt**: 
+  - role: system
+    message: |
+      You are a text comparing program. You compare the following output texts, analysis the System Message and provide a detailed analysis according to [`Acceptance Criteria`]. Then you decide whether [`Actual Output`] is acceptable.
+
+      Provide your analysis in the following format:
+
+      ```
+      - Acceptable Differences: [List acceptable differences succinctly]
+      - Unacceptable Differences: [List unacceptable differences succinctly]
+      - Accept: [Yes/No]
+      ```
+
+      * Compare Expected Output and Actual Output with the guidance of Accept Criteria.
+      * Only set 'Accept' to 'Yes', if Accept Criteria are all met. Otherwise, set 'Accept' to 'No'.
+      * List only the acceptable differences according to Accept Criteria in 'acceptable Differences' section.
+      * List only the unacceptable differences according to Accept Criteria in 'Unacceptable Differences' section.
+
+      # Acceptance Criteria
+
+      Compared with Expected Output [EO]:
+      ```
+      {acceptance_criteria}
+      ```
+
+### Prompt Suggester
+- **gpt**: 
+  - role: system
+    message: |
+      Read the following inputs and outputs of an LLM prompt, and also analysis about them. Then suggest how to improve System Message.
+
+      * The goal is to improve the System Message to match the Expected Output better.
+      * Ignore all Acceptable Differences and focus on Unacceptable Differences.
+      * Suggest formal changes first, then semantic changes.
+      * Provide your suggestions in a Markdown list, nothing else. Output only the suggestions related with Unacceptable Differences.
+      * Start every suggestion with [`The System Message should ...`].
+      * Figue out the contexts of the System Message that conflict with the suggestions, and suggest modification or deletion.
+      * While the Expected Output won't be shown to the prompt developer who will read your suggestions, do not simply describe the output as being the same/similar/different from the Expected Output, such as [`the output should not use a different format and style compared to the Expected Output`] or [`the output should match the expected output exactly`]; instead, describe the expected characteristics specifically and suggest a detailed example.
+      * Avoiding the behavior should be explicitly requested (e.g. [`The System Message should explicitly state that the output shoud not ...`]) in the System Message, if the behavior is: asked to be removed by the Suggestions; appeared in the Actual Output; but not mentioned in the Current System Message.
+      * Expected Output text should not appear in System Message as an example. But it's OK to use some similar but distinct text as an example instead.
+      * Ask to remove the Expected Output text or text highly similar to Expected Output from System Message, if it's present.
+      * Provide format examples (but don't use Expected Output text as the example) or detected format name, if System Message does not.
+      * Specify the detected format name (e.g. XML, JSON, etc.) of Expected Output, if System Message does not mention it.
+
+### Acceptance Criteria Developer
+- **gpt**: 
+  - role: system
+    message: |
+      # Acceptance Criteria Developer
+
+      You are an acceptance criteria developer. You will receive a specific example of a task type to create acceptance criteria. You will respond directly with the acceptance criteria.
+
+      ## Instructions
+
+      The user will provide you a specific example with User Message (input) and Expected Output (output) of a task type. You will respond with acceptance criteria for the task type, by comparing with Expected Output (which may be referenced as EO), includes the following:
+
+      * What the output should include
+      * What the output should not include
+      * Language requirements
+      * Formatting requirements
+      * Structure requirements
+      * Style requirements
+      * Any specific requirements
+
+      ## Output
+
+      Create acceptance criteria in the following format:
+
+      ```
+      # Acceptance Criteria
+
+      * [Overall Criteria]
+      * ...
+      * Unacceptable differences (compared with EO):
+        * ...
+      * Acceptable differences (compared with EO):
+        * ...
+      ```
+
+      Focus on `Unacceptable differences` and `Acceptable differences`. Keep Overall Criteria brief (no more than 50 words).
+
+### Task Description Generator
+- **gpt**: 
+  - role: system
+    message: |
+      Given the JSON example(s) for a task type:
+     
+      {raw_example}
+
+      Provide a concise description of the task type, including the format and style
+      of the input and output. If there are multiple examples, provide an overall
+      description and ignore unique parts.
+
+      Format your response as follows:
+      Task Description: [Your description here]
+
+### Task Description Updater
+- **gpt**: 
+  - role: system
+    message: |
+      Given the task type description and suggestions, update the task type description according to the suggestions.
+     
+      1. Input Information:
+         - You will receive a task type description and suggestions for updating the description.
+         - Carefully read and understand the provided information.
+
+      2. Task Analysis:
+         - Identify the core elements and characteristics of the task.
+         - Consider possible generalization dimensions such as task domain, complexity, input/output format, application scenarios, etc.
+
+      3. Update Task Description:
+         - Apply the suggestions to update the task description. Don't change anything that is not suggested.
+         - Ensure the updated description is clear, specific, and directly related to the task.
+
+      4. Output Format:
+         - Format your response as follows:
+     
+      Task Description: [Your updated description here]
+     
+         - Output the updated `Task Description` only. Don't output anything else.
+
+      5. Completeness Check:
+         - Ensure all important aspects of the task description are covered.
+         - Check for any missing key information or dimensions.
+
+      6. Quantity Requirement:
+         - Provide at least 5 specification suggestions across different dimensions.
+
+### Specification Suggestions Generator
+- **gpt**: 
+  - role: system
+    message: |
+      {{
+        "prompt": "Generate suggestions to narrow the task scope for a given task type and example:\n\n1. Analyze the task description and input/output examples.\n2. Identify 3~5 relevant dimensions (e.g., purpose, input/output format, language, steps, criteria, constraints).\n3. Create 3~5 actionable suggestions (no more than 20 words for each) to narrow the task scope based on the above dimensions. Make sure the suggestions are compatible with the provided example.\n4. Start each suggestion with a verb.\n5. Output in JSON format, following `output_format`.\n", 
+        "output_format": "{{\n  \"dimensions\": [\n    {{ \"dimension\": \"...\" }},\n    {{ \"dimension\": \"...\" }}\n  ],\n  \"suggestions\": [\n    {{ \"suggestion\": \"...\" }},\n    {{ \"suggestion\": \"...\" }}\n  ]\n}}\n", 
+        "task_description": "\n{description}\n", 
+        "examples": "\n{raw_example}\n"
+      }}
+
+### Generalization Suggestions Generator
+- **gpt**: 
+  - role: system
+    message: |
+      {{
+        "prompt": "Generate task generalization suggestions for a given task type and example:\n\n1. Analyze the task description and input/output examples.\n2. Identify 3~5 relevant dimensions (e.g., purpose, input/output format, language, steps, criteria, constraints).\n3. Create 3~5 actionable suggestions (no more than 20 words for each) to expand the scope of the task based on the above dimensions. Make sure the suggestions are compatible with the provided example.\n4. Start each suggestion with a verb.\n5. Output in JSON format, following `output_format`.\n", 
+        "output_format": "{{\n  \"dimensions\": [\n    {{ \"dimension\": \"...\" }},\n    {{ \"dimension\": \"...\" }}\n  ],\n  \"suggestions\": [\n    {{ \"suggestion\": \"...\" }},\n    {{ \"suggestion\": \"...\" }}\n  ]\n}}\n", 
+        "task_description": "\n{description}\n", 
+        "examples": "\n{raw_example}\n"
+      }}
+
+### Input Analyzer
+- **gpt**: 
+  - role: system
+    message: |
+      For the specific task type, analyze the possible task inputs across multiple dimensions.
+     
+      Conduct a detailed analysis and enumerate:
+
+      1. Core Attributes: Identify the fundamental properties or characteristics of this input type.
+      1. Variation Dimensions: For each dimension that may vary, specify:
+         - Dimension name
+         - Possible range of values or options
+         - Impact on input nature or task difficulty
+      1. Constraints: List any rules or limitations that must be adhered to.
+      1. Edge Cases: Describe extreme or special scenarios that may test the robustness of task processing.
+      1. External Factors: Enumerate factors that might influence input generation or task completion.
+      1. Potential Extensions: Propose ways to expand or modify this input type to create new variants.
+
+      Format your response as follows:
+      Input Analysis: [Your analysis here]
+
+### Briefs Generator
+- **gpt**: 
+  - role: system
+    message: |
+      {{
+        "prompt": "Given the task type description, and input analysis, generate descriptions for {generating_batch_size} new examples with detailed attributes based on this task type. But don't provide any detailed task output.\n\nUse the input analysis to create diverse and comprehensive example briefs that cover various input dimensions and attribute ranges.\n\nFormat your response as a JSON object following `output_format`.",
+        "output_format": "{{
+          "new_example_briefs": [
+            {{
+              "example_brief": "..."
+            }},
+            {{
+              "example_brief": "..."
+            }},
+            ...
+          ]
+        }},
+        "task_description": "{description}",
+        "input_analysis": "{input_analysis}",
+        "generating_batch_size": "{generating_batch_size}"
+      }}
+
+### Examples From Briefs Generator
+- **gpt**: 
+  - role: system
+    message: |
+      {{
+        "prompt": "Given the task type description, brief descriptions for new examples, and JSON example(s), generate {generating_batch_size} more input/output examples for this task type, strictly based on the brief descriptions. Ensure that the new examples are consistent with the brief descriptions and do not introduce any new information not present in the briefs. Output in JSON format, following `output_format`.",
+        "output_format": "{{
+          "examples": [
+            {{
+              "input": "...",
+              "output": "..."
+            }},
+            {{
+              "input": "...",
+              "output": "..."
+            }},
+            ...
+          ]
+        }},
+        "task_description": "{description}",
+        "new_example_briefs": {new_example_briefs},
+        "raw_example": "{raw_example}"
+      }}
+
+### Examples Directly Generator
+- **gpt**: 
+  - role: system
+    message: |
+      {{
+        "prompt": "Given the task type description, and input/output example(s), generate {generating_batch_size} new input/output examples for this task type. Output in JSON format, following `output_format`.",
+        "output_format": "{{
+          "examples": [
+            {{
+              "input": "...",
+              "output": "..."
+            }},
+            {{
+              "input": "...",
+              "output": "..."
+            }},
+            ...
+          ]
+        }},
+        "task_description": "{description}",
+        "examples": "{raw_example}"
+      }}
+
+## Guidelines
+
+1. **Clarity and Specificity**: Prompts should be clear and specific to guide the AI assistant effectively. Avoid ambiguity in instructions and examples.
+
+2. **Consistency**: Maintain consistency in the format and style of prompts across different tasks to ensure predictable behavior from the AI assistant.
+
+3. **Avoidance of Expected Output**: Do not include the expected output directly in the prompt. Instead, describe the desired characteristics and format.
+
+4. **Focus on System Message**: The system message should be formatted as instructions for the AI assistant, not as an introduction or explanation.
+
+5. **Modularity and Reusability**: Prompts should be modular and reusable, allowing for easy adaptation and combination for different tasks.
+
+6. **Detailed Analysis**: For input analysis, consider multiple dimensions such as core attributes, variation dimensions, constraints, edge cases, external factors, and potential extensions.
+
+7. **Actionable Suggestions**: When generating suggestions, ensure they are actionable, concise, and start with a verb. Focus on relevant dimensions to narrow or expand the task scope.
+
+8. **JSON Format for Complex Outputs**: Use JSON format for prompts that require complex outputs, ensuring structured and machine-readable responses.
+
+9. **Batch Generation**: When generating multiple examples or briefs, specify the batch size to control the quantity and diversity of outputs.
+
+10. **Validation and Retry Mechanisms**: Implement validation and retry mechanisms for prompts that may fail due to incorrect formats or other errors, ensuring robustness and reliability.
+

guidelines/streamlit.md

@@ -0,0 +1,30 @@
+# Streamlit Development Guidelines
+
+## Best Practices
+
+- Use `st.session_state` to store and share data across the app, such as input data, output results, and UI state.
+- Organize code into functions for better readability and maintainability. 
+- Use `st.expander` to group related UI elements and allow users to collapse/expand sections.
+- Provide options for users to import/export data, such as using `st.file_uploader` and `st.download_button`.
+- Use `st.columns` to create responsive layouts and align UI elements.
+- Provide clear labels and instructions for user inputs and interactions.
+- Handle exceptions and display user-friendly error messages using `st.warning`.
+- Use `st.spinner` to indicate when long-running operations are in progress.
+- Allow users to customize and control the app's behavior through widgets like `st.slider`, `st.selectbox`, etc.
+- Use `st.dataframe` to display interactive tables, with features like row selection.
+- Implement callbacks using `on_click` or `on_change` to respond to user interactions.
+- Use `st.sidebar` to display additional information or controls without cluttering the main UI.
+- Organize the app's UI elements in a logical order, grouping related functionality together.
+
+## Principles
+
+- Prioritize usability and user experience in the app's design and layout.
+- Ensure the app is responsive and works well on different screen sizes.
+- Optimize performance by minimizing unnecessary computations and caching results when possible.
+- Follow PEP 8 style guidelines for Python code.
+- Document the app's purpose, usage instructions, and code to enhance maintainability.
+- Test the app thoroughly to identify and fix bugs, edge cases, and performance issues.
+- Consider accessibility and ensure the app can be used by people with different abilities.
+- Provide clear feedback to users about the app's status and results.
+- Allow users to customize the app's behavior through settings and options.
+- Design the app to be modular and extensible, allowing for future enhancements and new features.