Displays logs. Optimized prompts.

Dockerfile

@@ -6,13 +6,15 @@ WORKDIR /app
 
 # Copy all files from the current directory to the working directory in the container
 COPY config.yml poetry.lock pyproject.toml /app/
-COPY app /app/app/
-COPY meta_prompt /app/meta_prompt/
 
 RUN pip install --no-cache-dir -U poetry
 RUN poetry config virtualenvs.create false
 RUN poetry install --with=dev
 
+COPY meta_prompt /app/meta_prompt/
+COPY app /app/app/
+RUN poetry install --with=dev
+
 # Expose the port (if necessary)
 EXPOSE 7860
 

app/config.py

@@ -16,4 +16,7 @@ class MetaPromptConfig(BaseConfig):
     server_port: Optional[int] = None
     recursion_limit: Optional[int] = 25
     recursion_limit_max: Optional[int] = 50
-    allow_flagging: Optional[bool] = False
+    allow_flagging: Optional[bool] = False
+    verbose: Optional[bool] = False
+    max_output_age: Optional[int] = 3
+    max_output_age_max: Optional[int] = 8

app/gradio_meta_prompt.py

@@ -1,15 +1,20 @@
 import csv
+import io
+import json
+import logging
 from pathlib import Path
+import pprint
 from typing import Any, Dict, Union
 import gradio as gr
-from gradio import CSVLogger, utils, Button
+from gradio import CSVLogger, Button, utils
 from gradio.flagging import FlagMethod
 from gradio_client import utils as client_utils
 from confz import BaseConfig, CLArgSource, EnvSource, FileSource
-from meta_prompt import MetaPromptGraph, AgentState
-from langchain_openai import ChatOpenAI
 from app.config import MetaPromptConfig
 from langchain_core.language_models import BaseLanguageModel
+from langchain_openai import ChatOpenAI
+from meta_prompt import MetaPromptGraph, AgentState
+from pythonjsonlogger import jsonlogger
 
 class SimplifiedCSVLogger(CSVLogger):
     """
@@ -70,19 +75,56 @@ class LLMModelFactory:
     
 llm_model_factory = LLMModelFactory()
 
+def chat_log_2_chatbot_list(chat_log: str):
+    chatbot_list = []
+    if chat_log is None or chat_log == '':
+        return chatbot_list
+    for line in chat_log.splitlines():
+        try:
+            json_line = json.loads(line)
+            if 'action' in json_line:
+                if json_line['action'] == 'invoke':
+                    chatbot_list.append([json_line['message'],None])
+                if json_line['action'] == 'response':
+                    chatbot_list.append([None,json_line['message']])
+        except json.decoder.JSONDecodeError as e:
+            print(f"Error decoding JSON log output: {e}")
+            print(line)
+        except KeyError as e:
+            print(f"Error accessing key in JSON log output: {e}")
+            print(line)
+    return chatbot_list
+
 def process_message(user_message, expected_output, acceptance_criteria, initial_system_message,
-                    recursion_limit: int, llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]]):
+                    recursion_limit: int, max_output_age: int,
+                    llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]]):
     # Create the input state
     input_state = AgentState(
         user_message=user_message,
         expected_output=expected_output,
         acceptance_criteria=acceptance_criteria,
-        system_message=initial_system_message
+        system_message=initial_system_message,
+        max_output_age=max_output_age
     )
     
     # Get the output state from MetaPromptGraph
-    meta_prompt_graph = MetaPromptGraph(llms=llms)
+    log_stream = io.StringIO()
+    log_handler = None
+    logger = None
+    if config.verbose:
+        log_handler = logging.StreamHandler(log_stream)
+        logger = logging.getLogger(MetaPromptGraph.__name__)
+        log_handler.setFormatter(jsonlogger.JsonFormatter('%(asctime)s %(name)s %(levelname)s %(message)s'))
+        logger.addHandler(log_handler)
+
+    meta_prompt_graph = MetaPromptGraph(llms=llms, verbose=config.verbose, logger=logger)
     output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
+
+    if config.verbose:
+        log_handler.close()
+        log_output = log_stream.getvalue()
+    else:
+        log_output = None
     
     # Validate the output state
     system_message = ''
@@ -104,21 +146,23 @@ def process_message(user_message, expected_output, acceptance_criteria, initial_
     else:
         analysis = "Error: The output state does not contain a valid 'analysis'"
 
-    return system_message, output, analysis
+    return system_message, output, analysis, chat_log_2_chatbot_list(log_output)
 
 
 def process_message_with_single_llm(user_message, expected_output, acceptance_criteria, initial_system_message,
-                                    recursion_limit: int, model_name: str):
+                                    recursion_limit: int, max_output_age: int,
+                                    model_name: str):
     # Get the output state from MetaPromptGraph
     type = config.llms[model_name].type
     args = config.llms[model_name].model_dump(exclude={'type'})
     llm = llm_model_factory.create(type, **args)
 
     return process_message(user_message, expected_output, acceptance_criteria, initial_system_message,
-                           recursion_limit, llm)
+                           recursion_limit, max_output_age, llm)
 
 def process_message_with_2_llms(user_message, expected_output, acceptance_criteria, initial_system_message,
-                                       recursion_limit: int, optimizer_model_name: str, executor_model_name: str,):
+                                       recursion_limit: int, max_output_age: int,
+                                       optimizer_model_name: str, executor_model_name: str,):
     # Get the output state from MetaPromptGraph
     optimizer_model = llm_model_factory.create(config.llms[optimizer_model_name].type,
                                                **config.llms[optimizer_model_name].model_dump(exclude={'type'}))
@@ -134,7 +178,7 @@ def process_message_with_2_llms(user_message, expected_output, acceptance_criter
     }
 
     return process_message(user_message, expected_output, acceptance_criteria, initial_system_message,
-                           recursion_limit, llms)
+                           recursion_limit, max_output_age, llms)
 
 class FileConfig(BaseConfig):
     config_file: str = 'config.yml'  # default path
@@ -151,20 +195,12 @@ config_sources = [
     CLArgSource()
 ]
 
-# Add event handlers
-def handle_submit(user_message, expected_output, acceptance_criteria, initial_system_message, recursion_limit, model_name):
-    return process_message(user_message, expected_output, acceptance_criteria, initial_system_message, recursion_limit, model_name)
-
-# Define clear function
-def clear_inputs():
-    return "", "", "", "", "", ""
-
 config = MetaPromptConfig(config_sources=config_sources)
 
 flagging_callback = SimplifiedCSVLogger()
 
 # Create a Gradio Blocks context
-with gr.Blocks() as demo:
+with gr.Blocks(title='Meta Prompt') as demo:
     # Define the layout
     with gr.Row():
         gr.Markdown(f"""<h1 style='text-align: left; margin-bottom: 1rem'>Meta Prompt</h1>
@@ -182,7 +218,8 @@ with gr.Blocks() as demo:
                 label="Initial System Message", show_copy_button=True, value="")
             recursion_limit_input = gr.Number(label="Recursion Limit", value=config.recursion_limit,
                                               precision=0, minimum=1, maximum=config.recursion_limit_max, step=1)
-
+            max_output_age = gr.Number(label="Max Output Age", value=config.max_output_age,
+                                       precision=0, minimum=1, maximum=config.max_output_age_max, step=1)
             with gr.Row():
                 with gr.Tab('Simple'):
                     model_name_input = gr.Dropdown(
@@ -193,7 +230,10 @@ with gr.Blocks() as demo:
                     # Connect the inputs and outputs to the function
                     with gr.Row():
                         submit_button = gr.Button(value="Submit", variant="primary")
-                        clear_button = gr.Button(value="Clear", variant="secondary")
+                        clear_button = gr.ClearButton(
+                            [user_message_input, expected_output_input,
+                             acceptance_criteria_input, initial_system_message_input],
+                            value='Clear All')
                 with gr.Tab('Advanced'):
                     optimizer_model_name_input = gr.Dropdown(
                         label="Optimizer Model Name",
@@ -208,7 +248,11 @@ with gr.Blocks() as demo:
                     # Connect the inputs and outputs to the function
                     with gr.Row():
                         multiple_submit_button = gr.Button(value="Submit", variant="primary")
-                        multiple_clear_button = gr.Button(value="Clear", variant="secondary")
+                        multiple_clear_button = gr.ClearButton(components=[user_message_input,
+                                                                           expected_output_input,
+                                                                           acceptance_criteria_input,
+                                                                           initial_system_message_input],
+                                                               value='Clear All')
         with gr.Column():
             system_message_output = gr.Textbox(
                 label="System Message", show_copy_button=True)
@@ -216,20 +260,26 @@ with gr.Blocks() as demo:
             analysis_output = gr.Textbox(
                 label="Analysis", show_copy_button=True)
             flag_button = gr.Button(value="Flag", variant="secondary", visible=config.allow_flagging)
+            with gr.Accordion("Details", open=False, visible=config.verbose):
+                logs_chatbot = gr.Chatbot(label='Messages', show_copy_button=True, layout='bubble',
+                                          bubble_full_width=False, render_markdown=False)
+                clear_logs_button = gr.ClearButton([logs_chatbot], value='Clear Logs')
+
+        clear_button.add([system_message_output, output_output,
+                         analysis_output, logs_chatbot])
+        multiple_clear_button.add([system_message_output, output_output,
+                                    analysis_output, logs_chatbot])
 
     submit_button.click(process_message_with_single_llm,
                         inputs=[user_message_input, expected_output_input, acceptance_criteria_input,
-                                initial_system_message_input, recursion_limit_input, model_name_input],
-                        outputs=[system_message_output, output_output, analysis_output])
-    clear_button.click(clear_inputs,
-                        outputs=[user_message_input, expected_output_input, acceptance_criteria_input, initial_system_message_input])
+                                initial_system_message_input, recursion_limit_input, max_output_age,
+                                model_name_input],
+                        outputs=[system_message_output, output_output, analysis_output, logs_chatbot])
     multiple_submit_button.click(process_message_with_2_llms,
                                     inputs=[user_message_input, expected_output_input, acceptance_criteria_input,
-                                            initial_system_message_input, recursion_limit_input,
+                                            initial_system_message_input, recursion_limit_input, max_output_age,
                                             optimizer_model_name_input, executor_model_name_input],
-                                    outputs=[system_message_output, output_output, analysis_output])
-    multiple_clear_button.click(clear_inputs,
-                                outputs=[user_message_input, expected_output_input, acceptance_criteria_input, initial_system_message_input])
+                                    outputs=[system_message_output, output_output, analysis_output, logs_chatbot])
 
     # Load examples
     examples = config.examples_path

config.yml

@@ -27,7 +27,7 @@ llms:
     model_name: "mixtral-8x7b-32768"
     openai_api_base: "https://api.groq.com/openai/v1"
     max_tokens: 8192
-    verbose: true  
+    verbose: true
   # anthropic/claude-3-haiku:
   #   type: ChatOpenAI
   #   temperature: 0.1
@@ -58,4 +58,5 @@ server_name: 0.0.0.0
 # server_port: 7860
 recursion_limit: 16
 recursion_limit_max: 20
-allow_flask_cors: false
+allow_flagging: false
+verbose: false

meta_prompt/meta_prompt.py

@@ -77,11 +77,14 @@ Output and match Expected Output more closely.
 3. Modify only the content mentioned in the Suggestion. Do not change the
    parts that are not related to the Suggestion.
 4. Output only the updated system message, without any additional content.
-5. Expected Output text should not appear in System Message as an example. But
+5. Avoiding the behavior should be explicitly requested (e.g. `Don't ...`) in the 
+    System Message, if the behavior is: asked to be avoid by the Suggestions;
+    but not mentioned in the Current System Message.
+6. Expected Output text should not appear in System Message as an example. But
    it's OK to use some similar text as an example instead.
    * Remove the Expected Output text or text highly similar to Expected Output
      from System Message, if it's present.
-6. Format the system message well, with no more than 80 characters per line
+7. Format the system message well, with no more than 80 characters per line
    (except for raw text).
 
 ## Output
@@ -112,12 +115,13 @@ Provide only the updated System Message, adhering to the above guidelines.
         NODE_OUTPUT_HISTORY_ANALYZER: ChatPromptTemplate.from_messages([
             ("system", """You are a text comparing program. You read the Acceptance Criteria, compare the
 compare the exptected output with two different outputs, and decide which one is
-more similar to the expected output.
+more consistent with the expected output. When comparing the outputs, ignore the
+differences which are acceptable or ignorable according to the Acceptance Criteria.
 
 You output the following analysis according to the Acceptance Criteria:
 
 * Your analysis in a Markdown list.
-* The ID of the output that is more similar to the Expected Output as Preferred
+* The ID of the output that is more consistent with the Expected Output as Preferred
     Output ID, with the following format:
     
 ```
@@ -210,50 +214,52 @@ Provide your analysis in the following format:
         NODE_PROMPT_SUGGESTER: ChatPromptTemplate.from_messages([
             ("system", """
 Read the following inputs and outputs of an LLM prompt, and also analysis about them.
-Then suggest how to improve System Prompt.
+Then suggest how to improve System Message.
 
-System Prompt:
-```
-{system_message}
-```
-User Message:
-```
-{user_message}
-```
-Expected Output: 
-```
-{expected_output}
-```
-Actual Output: 
-```
-{output}
-```
-
-Acceptance Criteria:
-```
-{acceptance_criteria}
-```
-
-Analysis:
-```
-{analysis}
-```
-
-* The goal is to improve the System Prompt to match the Expected Output better.
+* The goal is to improve the System Message to match the Expected Output better.
 * Ignore all Acceptable Differences and focus on Unacceptable Differences.
 * Suggest formal changes first, then semantic changes.
 * Provide your suggestions in a Markdown list, nothing else. Output only the
     suggestions related with Unacceptable Differences.
-    * Use `... should ...` to clearly state the desired output.
+    * Start every suggestion with `The System Message should ...`.
     * Figue out the contexts of the System Message that conflict with the suggestions,
     and suggest modification or deletion.
+    * Avoiding the behavior should be explicitly requested (e.g. `The System Message 
+    should explicitly state that the output shoud not ...`) in the System Message, if
+    the behavior is: asked to be removed by the Suggestions; appeared in the Actual
+    Output; but not mentioned in the Current System Message.
 * Expected Output text should not appear in System Message as an example. But
-    it's OK to use some similar text as an example instead.
+    it's OK to use some similar but distinct text as an example instead.
     * Ask to remove the Expected Output text or text highly similar to Expected Output
     from System Message, if it's present.
 * Provide format examples or detected format name, if System Message does not.
     * Specify the detected format name (e.g. XML, JSON, etc.) of Expected Output, if
     System Message does not mention it.
+"""),
+            ("human", """
+<|Start_System_Message|>
+{system_message}
+<|End_System_Message|>
+
+<|Start_User_Message|>
+{user_message}
+<|End_User_Message|>
+
+<|Start_Expected_Output|>
+{expected_output}
+<|End_Expected_Output|>
+
+<|Start_Actual_Output|>
+{output}
+<|End_Actual_Output|>
+
+<|Start_Acceptance Criteria|>
+{acceptance_criteria}
+<|End_Acceptance Criteria|>
+
+<|Start_Analysis|>
+{analysis}
+<|End_Analysis|>
 """)
         ])
     }
@@ -272,12 +278,14 @@ Analysis:
     def __init__(self,
                  llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]] = {},
                  prompts: Dict[str, ChatPromptTemplate] = {},
+                 logger: Optional[logging.Logger] = None,
                  verbose = False):
-        self.logger = logging.getLogger(__name__)
-        if verbose:
-            self.logger.setLevel(logging.DEBUG)
-        else:
-            self.logger.setLevel(logging.INFO)
+        self.logger = logger or logging.getLogger(__name__)
+        if self.logger is not None:
+            if verbose:
+                self.logger.setLevel(logging.DEBUG)
+            else:
+                self.logger.setLevel(logging.INFO)
 
         if isinstance(llms, BaseLanguageModel):
             self.llms: Dict[str, BaseLanguageModel] = {node: llms for node in self.get_node_names()}
@@ -374,32 +382,36 @@ Analysis:
         return state
 
     def _prompt_node(self, node, target_attribute: str, state: AgentState) -> AgentState:
+        logger = self.logger.getChild(node)
         prompt = self.prompt_templates[node].format_messages(**state.model_dump())
 
-        self.logger.debug("Invoking %s with prompt: %s", node, pprint.pformat(prompt))
+        for message in prompt:
+            logger.debug({'node': node, 'action': 'invoke', 'type': message.type, 'message': message.content})
         response = self.llms[node].invoke(self.prompt_templates[node].format_messages(**state.model_dump()))
-        self.logger.debug("Response: %s", response.content)
+        logger.debug({'node': node, 'action': 'response', 'type': response.type, 'message': response.content})
         
         setattr(state, target_attribute, response.content)
         return state
 
     def _output_history_analyzer(self, state: AgentState) -> AgentState:
+        logger = self.logger.getChild(self.NODE_OUTPUT_HISTORY_ANALYZER)
+
         if state.best_output is None:
             state.best_output = state.output
             state.best_system_message = state.system_message
             state.best_output_age = 0
 
-            self.logger.debug("Best output initialized to the current output: \n %s", state.output)
+            logger.debug("Best output initialized to the current output: \n %s", state.output)
 
             return state
 
         prompt = self.prompt_templates[self.NODE_OUTPUT_HISTORY_ANALYZER].format_messages(**state.model_dump())
 
-        self.logger.debug("Invoking %s with prompt: %s",
-                          self.NODE_OUTPUT_HISTORY_ANALYZER, 
-                          pprint.pformat(prompt))
+        for message in prompt:
+            logger.debug({'node': self.NODE_OUTPUT_HISTORY_ANALYZER, 'action': 'invoke', 'type': message.type, 'message': message.content})
+
         response = self.llms[self.NODE_OUTPUT_HISTORY_ANALYZER].invoke(prompt)
-        self.logger.debug("Response: %s", response.content)
+        logger.debug({'node': self.NODE_OUTPUT_HISTORY_ANALYZER, 'action': 'response', 'type': response.type, 'message': response.content})
 
         analysis = response.content
 
@@ -408,35 +420,44 @@ Analysis:
             state.best_system_message = state.system_message
             state.best_output_age = 0
 
-            self.logger.debug("Best output updated to the current output: \n %s", state.output)
+            logger.debug("Best output updated to the current output: \n %s", state.output)
         else:
             state.best_output_age += 1
 
-            self.logger.debug("Best output age incremented to %s", state.best_output_age)
+            logger.debug("Best output age incremented to %s", state.best_output_age)
 
         return state
 
     def _prompt_analyzer(self, state: AgentState) -> AgentState:
+        logger = self.logger.getChild(self.NODE_PROMPT_ANALYZER)
         prompt = self.prompt_templates[self.NODE_PROMPT_ANALYZER].format_messages(**state.model_dump())
 
-        self.logger.debug("Invoking %s with prompt: %s",
-                          self.NODE_PROMPT_ANALYZER,
-                          pprint.pformat(prompt))
+        for message in prompt:
+            logger.debug({'node': self.NODE_PROMPT_ANALYZER, 'action': 'invoke', 'type': message.type, 'message': message.content})
+
         response = self.llms[self.NODE_PROMPT_ANALYZER].invoke(prompt)
-        self.logger.debug("Response: %s", response.content)
+        logger.debug({'node': self.NODE_PROMPT_ANALYZER, 'action': 'response', 'type': response.type, 'message': response.content})
 
         state.analysis = response.content
         state.accepted = "Accept: Yes" in response.content
 
-        self.logger.debug("Accepted: %s", state.accepted)
+        logger.debug("Accepted: %s", state.accepted)
 
         return state
 
     def _should_exit_on_max_age(self, state: AgentState) -> str:
-        if state.max_output_age <= 0 or state.best_output_age < state.max_output_age:
+        if state.max_output_age <=0:
+            # always continue if max age is 0
             return "continue"
-        else:
-            return "rerun"
+        
+        if state.best_output_age >= state.max_output_age:
+            return END
+        
+        if state.best_output_age > 0:
+            # skip prompt_analyzer and prompt_suggester, goto prompt_developer
+            return "rerun" 
+        
+        return "continue"
 
     def _should_exit_on_acceptable_output(self, state: AgentState) -> str:
         return "continue" if not state.accepted else END

poetry.lock

@@ -2160,6 +2160,17 @@ files = [
 [package.extras]
 cli = ["click (>=5.0)"]
 
+[[package]]
+name = "python-json-logger"
+version = "2.0.7"
+description = "A python library adding a json log formatter"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "python-json-logger-2.0.7.tar.gz", hash = "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c"},
+    {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"},
+]
+
 [[package]]
 name = "python-multipart"
 version = "0.0.9"
@@ -3290,4 +3301,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "e4535633af687ad735ec90d956f750dbb4100d0e24ad350a8f25c20247b43444"
+content-hash = "dca41322d7cd0e10cb5cd3e5748b9f505a2d29f00ca78816e397c1c1f5e9d693"

pyproject.toml

@@ -11,6 +11,7 @@ langgraph = "^0.1.5"
 langchain = "^0.2.6"
 langchain-openai = "^0.1.14"
 pydantic = "^2.8.2"
+python-json-logger = "^2.0.7"
 
 [tool.poetry.dev-dependencies]
 gradio = "^4.37.2"